diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 921e07cc..c80d6d66 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -6,6 +6,11 @@ website 'name': 'AdGuard filters issues' 'url': 'https://link.adtidy.org/forward.html?action=report&app=home&from=github' + - 'about': > + Please send requests for addition to the vetted filtering lists to the + Hostlists Registry repository. + 'name': 'AdGuard Hostlists Registry' + 'url': 'https://github.com/AdguardTeam/HostlistsRegistry' - 'about': > Please use GitHub Discussions for questions 'name': 'Q&A Discussions' diff --git a/client/src/components/Filters/DnsBlocklist.js b/client/src/components/Filters/DnsBlocklist.js index 4059b48f..c4facadc 100644 --- a/client/src/components/Filters/DnsBlocklist.js +++ b/client/src/components/Filters/DnsBlocklist.js @@ -15,7 +15,7 @@ import { getObjDiff, } from '../../helpers/helpers'; -const filtersCatalog = require('../../helpers/filters/filters.json'); +import filtersCatalog from '../../helpers/filters/filters'; class DnsBlocklist extends Component { componentDidMount() { diff --git a/client/src/components/Filters/Form.js b/client/src/components/Filters/Form.js index 39791a5f..f4e902f5 100644 --- a/client/src/components/Filters/Form.js +++ b/client/src/components/Filters/Form.js @@ -7,8 +7,7 @@ import classNames from 'classnames'; import { validatePath, validateRequiredValue } from '../../helpers/validators'; import { CheckboxField, renderInputField } from '../../helpers/form'; import { MODAL_OPEN_TIMEOUT, MODAL_TYPE, FORM_NAME } from '../../helpers/constants'; - -const filtersCatalog = require('../../helpers/filters/filters.json'); +import filtersCatalog from '../../helpers/filters/filters'; const getIconsData = (homepage, source) => ([ { diff --git a/client/src/helpers/filters/filters.json b/client/src/helpers/filters/filters.js similarity index 59% rename from client/src/helpers/filters/filters.json rename to client/src/helpers/filters/filters.js index 63fe0995..8be075e9 100644 --- a/client/src/helpers/filters/filters.json +++ b/client/src/helpers/filters/filters.js @@ -1,162 +1,220 @@ -{ +// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT. + +/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */ + +export default { "categories": { "general": { "name": "filter_category_general", "description": "filter_category_general_desc" }, - "security": { - "name": "filter_category_security", - "description": "filter_category_security_desc" + "other": { + "name": "filter_category_other", + "description": "filter_category_other_desc" }, "regional": { "name": "filter_category_regional", "description": "filter_category_regional_desc" }, - "other": { - "name": "filter_category_other", - "description": "filter_category_other_desc" + "security": { + "name": "filter_category_security", + "description": "filter_category_security_desc" } }, "filters": { - "adguard-dns-filter": { - "name": "AdGuard DNS filter", + "1hosts_lite": { + "name": "1Hosts (Lite)", "categoryId": "general", - "homepage": "https://github.com/AdguardTeam/AdGuardSDNSFilter", - "source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt" + "homepage": "https://badmojr.github.io/1Hosts/", + "source": "https://badmojr.gitlab.io/1hosts/Lite/adblock.txt" }, - "adaway-default-blocklist": { - "name": "AdAway Default Blocklist", - "categoryId": "general", - "homepage": "https://github.com/AdAway/adaway.github.io/", - "source": "https://adaway.org/hosts.txt" - }, - "peter-lowe-list": { - "name": "Peter Lowe's List", - "categoryId": "general", - "homepage": "https://pgl.yoyo.org/adservers/", - "source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus&showintro=1&mimetype=plaintext" - }, - "dan-pollock-list": { - "name": "Dan Pollock's List", - "categoryId": "general", - "homepage": "https://someonewhocares.org/", - "source": "https://someonewhocares.org/hosts/zero/hosts" - }, - "oisd": { - "name": "OISD Blocklist Basic", - "categoryId": "general", - "homepage": "https://oisd.nl/", - "source": "https://abp.oisd.nl/basic/" - }, - "game-console-adblock-list": { - "name": "Game Console Adblock List", - "categoryId": "general", - "homepage": "https://github.com/DandelionSprout/adfilt", - "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt" - }, - "perflyst-dandelion-sprout-smart-tv-blocklist-for-adguard-home": { - "name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist", - "categoryId": "general", - "homepage": "https://github.com/Perflyst/PiHoleBlocklist", - "source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt" - }, - "nocoin-filter-list": { - "name": "NoCoin Filter List", - "categoryId": "security", - "homepage": "https://github.com/hoshsadiq/adblock-nocoin-list/", - "source": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt" - }, - "the-big-list-of-hacked-malware-web-sites": { - "name": "The Big List of Hacked Malware Web Sites", - "categoryId": "security", - "homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites", - "source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts" - }, - "scam-blocklist-by-durable-napkin": { - "name": "Scam Blocklist by DurableNapkin", - "categoryId": "security", - "homepage": "https://github.com/durablenapkin/scamblocklist", - "source": "https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/adguard.txt" - }, - "urlhaus-filter-online": { - "name": "Online Malicious URL Blocklist", - "categoryId": "security", - "homepage": "https://gitlab.com/malware-filter/urlhaus-filter", - "source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt" - }, - "dandelion-sprouts-anti-malware-list": { - "name": "Dandelion Sprout's Anti-Malware List", - "categoryId": "security", - "homepage": "https://github.com/DandelionSprout/adfilt", - "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt" - }, - "NOR-dandelion-sprouts-nordiske-filtre": { - "name": "NOR: Dandelion Sprouts nordiske filtre", + "CHN_adrules": { + "name": "CHN: AdRules DNS List", "categoryId": "regional", - "homepage": "https://github.com/DandelionSprout/adfilt", - "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt" + "homepage": "https://github.com/Cats-Team/AdRules", + "source": "https://raw.githubusercontent.com/Cats-Team/AdRules/main/dns.txt" }, - "POL-polish-filters-for-pihole": { - "name": "POL: Polish filters for Pi hole", - "categoryId": "regional", - "homepage": "https://www.certyficate.it/", - "source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt" - }, - "KOR-youslist": { - "name": "KOR: YousList", - "categoryId": "regional", - "homepage": "https://github.com/yous/YousList", - "source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt" - }, - "VNM-abpvn-list": { - "name": "VNM: ABPVN List", - "categoryId": "regional", - "homepage": "http://abpvn.com/", - "source": "https://abpvn.com/android/abpvn.txt" - }, - "SWE-frellwit-swedish-hosts-file": { - "name": "SWE: Frellwit's Swedish Hosts File", - "categoryId": "regional", - "homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/", - "source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt" - }, - "ITA-filtri-dns": { - "name": "ITA: Filtri-DNS", - "categoryId": "regional", - "homepage": "https://filtri-dns.ga/", - "source": "https://filtri-dns.ga/filtri.txt" - }, - "IRN-unwanted-iranian-domains": { - "name": "IRN: Unwanted Iranian domains", - "categoryId": "regional", - "homepage": "https://github.com/DRSDavidSoft/additional-hosts", - "source": "https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/unwanted-iranian.txt" - }, - "MKD-macedonian-pi-hole-blocklist": { - "name": "MKD: Macedonian Pi-hole Blocklist", - "categoryId": "regional", - "homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist", - "source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt" - }, - "CHN-anti-ad" : { + "CHN_anti_ad": { "name": "CHN: anti-AD", "categoryId": "regional", "homepage": "https://anti-ad.net/", "source": "https://anti-ad.net/easylist.txt" }, - "IDN-abpindo": { + "IDN_abpindo": { "name": "IDN: ABPindo", "categoryId": "regional", - "homepage": "https://github.com/ABPindo/indonesianadblockrules/", - "source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt" + "homepage": "https://github.com/ABPindo/indonesianadblockrules", + "source": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/aghome.txt" }, - "NLD-Easylist": { - "name": "NLD: Easylist", + "IRN_unwanted_iranian_domains": { + "name": "IRN: PersianBlocker list", "categoryId": "regional", - "homepage": "https://forums.lanik.us/viewforum.php?f=100", - "source": "https://easylist-downloads.adblockplus.org/easylistdutch.txt" + "homepage": "https://github.com/MasterKia/PersianBlocker", + "source": "https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt" }, - "windows-spy-blocker" : { + "ITA_filtri_dns": { + "name": "ITA: Filtri-DNS", + "categoryId": "regional", + "homepage": "https://filtri-dns.ga/", + "source": "https://filtri-dns.ga/filtri.txt" + }, + "KOR_list_kr": { + "name": "KOR: List-KR DNS", + "categoryId": "regional", + "homepage": "https://github.com/List-KR/List-KR", + "source": "https://github.com/List-KR/List-KR" + }, + "KOR_youslist": { + "name": "KOR: YousList", + "categoryId": "regional", + "homepage": "https://github.com/yous/YousList", + "source": "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt" + }, + "MKD_macedonian_pi_hole_blocklist": { + "name": "MKD: Macedonian Pi-hole Blocklist", + "categoryId": "regional", + "homepage": "https://github.com/cchevy/macedonian-pi-hole-blocklist", + "source": "https://raw.githubusercontent.com/cchevy/macedonian-pi-hole-blocklist/master/hosts.txt" + }, + "NOR_dandelion_sprouts_anti_malware_list": { + "name": "NOR: Dandelion Sprouts nordiske filtre", + "categoryId": "regional", + "homepage": "https://github.com/DandelionSprout/adfilt", + "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianExperimentalList%20alternate%20versions/NordicFiltersAdGuardHome.txt" + }, + "POL_polish_filters_for_pi_hole": { + "name": "POL: Polish filters for Pi hole", + "categoryId": "regional", + "homepage": "https://www.certyficate.it/", + "source": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt" + }, + "SWE_frellwit_swedish_hosts_file": { + "name": "SWE: Frellwit's Swedish Hosts File", + "categoryId": "regional", + "homepage": "https://github.com/lassekongo83/Frellwits-filter-lists/", + "source": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt" + }, + "TUR_turk_adlist": { + "name": "TUR: turk-adlist", + "categoryId": "regional", + "homepage": "https://github.com/bkrucarci/turk-adlist", + "source": "https://raw.githubusercontent.com/bkrucarci/turk-adlist/master/hosts" + }, + "VNM_abpvn": { + "name": "VNM: ABPVN List", + "categoryId": "regional", + "homepage": "http://abpvn.com/", + "source": "https://abpvn.com/android/abpvn.txt" + }, + "adguard_dns_filter": { + "name": "AdGuard DNS filter", + "categoryId": "general", + "homepage": "https://github.com/AdguardTeam/AdGuardSDNSFilter", + "source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt" + }, + "adway_default_blocklist": { + "name": "AdAway Default Blocklist", + "categoryId": "general", + "homepage": "https://github.com/AdAway/adaway.github.io/", + "source": "https://adaway.org/hosts.txt" + }, + "curben_phishing_filter": { + "name": "Phishing URL Blocklist (PhishTank and OpenPhish)", + "categoryId": "security", + "homepage": "https://gitlab.com/malware-filter/phishing-filter", + "source": "https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt" + }, + "dan_pollocks_list": { + "name": "Dan Pollock's List", + "categoryId": "general", + "homepage": "https://someonewhocares.org/", + "source": "https://someonewhocares.org/hosts/zero/hosts" + }, + "dandelion_sprouts_anti_malware_list": { + "name": "Dandelion Sprout's Anti-Malware List", + "categoryId": "security", + "homepage": "https://github.com/DandelionSprout/adfilt", + "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareAdGuardHome.txt" + }, + "dandelion_sprouts_game_console_adblock_list": { + "name": "Dandelion Sprout's Game Console Adblock List", + "categoryId": "other", + "homepage": "https://github.com/DandelionSprout/adfilt", + "source": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt" + }, + "energized_spark": { + "name": "Energized Spark", + "categoryId": "general", + "homepage": "https://energized.pro/", + "source": "https://block.energized.pro/spark/formats/filter" + }, + "nocoin_filter_list": { + "name": "NoCoin Filter List", + "categoryId": "security", + "homepage": "https://github.com/hoshsadiq/adblock-nocoin-list/", + "source": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt" + }, + "notracking_hosts_blocklists": { + "name": "The NoTracking blocklist", + "categoryId": "general", + "homepage": "https://github.com/notracking/hosts-blocklists", + "source": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/adblock/adblock.txt" + }, + "oisd_basic": { + "name": "OISD Blocklist Basic", + "categoryId": "general", + "homepage": "https://oisd.nl/", + "source": "https://abp.oisd.nl/basic/" + }, + "oisd_full": { + "name": "OISD Blocklist Full", + "categoryId": "general", + "homepage": "https://oisd.nl/", + "source": "https://abp.oisd.nl/" + }, + "perflyst_dandelion_sprout_smart_tv_blocklist_for_adguard_home": { + "name": "Perflyst and Dandelion Sprout's Smart-TV Blocklist", + "categoryId": "other", + "homepage": "https://github.com/Perflyst/PiHoleBlocklist", + "source": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt" + }, + "peter_lowe_list": { + "name": "Peter Lowe's Blocklist", + "categoryId": "general", + "homepage": "https://pgl.yoyo.org/adservers/", + "source": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus\u0026showintro=1\u0026mimetype=plaintext" + }, + "scam_blocklist_by_durablenapkin": { + "name": "Scam Blocklist by DurableNapkin", + "categoryId": "security", + "homepage": "https://github.com/durablenapkin/scamblocklist", + "source": "https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/adguard.txt" + }, + "staklerware_indicators_list": { + "name": "Stalkerware Indicators List", + "categoryId": "security", + "homepage": "https://github.com/AssoEchap/stalkerware-indicators", + "source": "https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/hosts" + }, + "steven_blacks_list": { + "name": "Steven Black's List", + "categoryId": "general", + "homepage": "https://github.com/StevenBlack/hosts", + "source": "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" + }, + "the_big_list_of_hacked_malware_web_sites": { + "name": "The Big List of Hacked Malware Web Sites", + "categoryId": "security", + "homepage": "https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites", + "source": "https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hosts" + }, + "urlhaus_filter_online": { + "name": "Malicious URL Blocklist (URLHaus)", + "categoryId": "security", + "homepage": "https://gitlab.com/malware-filter/urlhaus-filter", + "source": "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-agh.txt" + }, + "windowsspyblocker_hosts_spy_rules": { "name": "WindowsSpyBlocker - Hosts spy rules", "categoryId": "other", "homepage": "https://github.com/crazy-max/WindowsSpyBlocker", diff --git a/scripts/README.md b/scripts/README.md index 2ec087e0..3c9026b1 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -205,3 +205,23 @@ code from [the repo][companiesrepo]. ``` [companiesrepo]: https://github.com/AdguardTeam/companiesdb + + + +## `vetted-filters/`: Vetted Filters Updater + +A simple script that downloads and updates the vetted filtering list data from +AdGuard's [Hostlists Registry][reg]. + +Optional environment: + + * `URL`: the URL of the index file. By default it's + `https://adguardteam.github.io/HostlistsRegistry/assets/filters.json`. + + ### Usage + +```sh +go run ./scripts/vetted-filters/main.go +``` + +[reg]: https://github.com/AdguardTeam/HostlistsRegistry diff --git a/scripts/make/go-lint.sh b/scripts/make/go-lint.sh index 8c462d5b..31aba8c3 100644 --- a/scripts/make/go-lint.sh +++ b/scripts/make/go-lint.sh @@ -229,7 +229,7 @@ gocyclo --over 13 ./internal/dhcpd ./internal/filtering/ ./internal/home/ gocyclo --over 10 ./internal/aghio/ ./internal/aghnet/ ./internal/aghos/\ ./internal/aghtest/ ./internal/dnsforward/ ./internal/stats/\ ./internal/tools/ ./internal/updater/ ./internal/next/ ./internal/version/\ - ./main.go + ./scripts/vetted-filters/ ./main.go ineffassign ./... diff --git a/scripts/vetted-filters/main.go b/scripts/vetted-filters/main.go new file mode 100644 index 00000000..444e33a2 --- /dev/null +++ b/scripts/vetted-filters/main.go @@ -0,0 +1,164 @@ +// vetted-filters fetches the most recent Hostlists Registry index and +// transforms the filters from it to AdGuard Home's format. +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "time" + + "github.com/AdguardTeam/golibs/log" + "github.com/google/renameio/maybe" +) + +func main() { + urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json" + if v, ok := os.LookupEnv("URL"); ok { + urlStr = v + } + + // Validate the URL. + _, err := url.Parse(urlStr) + check(err) + + c := &http.Client{ + Timeout: 10 * time.Second, + } + + resp, err := c.Get(urlStr) + check(err) + defer log.OnCloserError(resp.Body, log.ERROR) + + if resp.StatusCode != http.StatusOK { + panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode)) + } + + hlFlt := &hlFilters{} + err = json.NewDecoder(resp.Body).Decode(hlFlt) + check(err) + + aghFlt := &aghFilters{ + Categories: map[string]*aghFiltersCategory{ + "general": { + Name: "filter_category_general", + Description: "filter_category_general_desc", + }, + "other": { + Name: "filter_category_other", + Description: "filter_category_other_desc", + }, + "regional": { + Name: "filter_category_regional", + Description: "filter_category_regional_desc", + }, + "security": { + Name: "filter_category_security", + Description: "filter_category_security_desc", + }, + }, + Filters: map[string]*aghFiltersFilter{}, + } + + for i, f := range hlFlt.Filters { + id := f.FilterID + cat := f.category() + if cat == "" { + log.Info("warning: filter %s at index %d does not have a fitting category", id, i) + } + + aghFlt.Filters[id] = &aghFiltersFilter{ + Name: f.Name, + CategoryID: cat, + Homepage: f.Homepage, + Source: f.SourceURL, + } + } + + buf := &bytes.Buffer{} + _, _ = buf.WriteString(jsHeader) + + enc := json.NewEncoder(buf) + enc.SetIndent("", " ") + + err = enc.Encode(aghFlt) + check(err) + + err = maybe.WriteFile("client/src/helpers/filters/filters.js", buf.Bytes(), 0o644) + check(err) +} + +// jsHeader is the header for the generates JavaScript file. It informs the +// reader that the file is generated and disables some style-related eslint +// checks. +const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT. + +/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */ + +export default ` + +// check is a simple error-checking helper for scripts. +func check(err error) { + if err != nil { + panic(err) + } +} + +// hlFilters is the JSON structure for the Hostlists Registry rule list index. +type hlFilters struct { + Filters []*hlFiltersFilter `json:"filters"` +} + +// hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry. +type hlFiltersFilter struct { + FilterID string `json:"filterId"` + Name string `json:"name"` + Homepage string `json:"homepage"` + SourceURL string `json:"sourceUrl"` + Tags []string `json:"tags"` +} + +// category returns the AdGuard Home category for this filter. If there is no +// fitting category, cat is empty. +func (f *hlFiltersFilter) category() (cat string) { + for _, t := range f.Tags { + switch t { + case "purpose:general": + return "general" + case "purpose:other": + return "other" + case "purpose:regional": + return "regional" + case "purpose:security": + return "security" + } + } + + return "" +} + +// aghFilters is the JSON structure for AdGuard Home's list of vetted filtering +// rule list in file client/src/helpers/filters/filters.js. +type aghFilters struct { + Categories map[string]*aghFiltersCategory `json:"categories"` + Filters map[string]*aghFiltersFilter `json:"filters"` +} + +// aghFiltersCategory is the JSON structure for a category in the vetted +// filtering rule list file. +type aghFiltersCategory struct { + Name string `json:"name"` + Description string `json:"description"` +} + +// aghFiltersFilter is the JSON structure for a filter in the vetted filtering +// rule list file. +type aghFiltersFilter struct { + Name string `json:"name"` + CategoryID string `json:"categoryId"` + Homepage string `json:"homepage"` + Source string `json:"source"` +}