Merge branch 'master' into websvc-confin-manager

This commit is contained in:
Ainar Garipov
2022-08-31 19:09:48 +03:00
17 changed files with 190 additions and 209 deletions

View File

@@ -34,7 +34,6 @@ func newARPDB() (arp *cmdARPDB) {
// The expected input format: // The expected input format:
// //
// host.name (192.168.0.1) at ff:ff:ff:ff:ff:ff on en0 ifscope [ethernet] // host.name (192.168.0.1) at ff:ff:ff:ff:ff:ff on en0 ifscope [ethernet]
//
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) { func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint) ns = make([]Neighbor, 0, lenHint)
for sc.Scan() { for sc.Scan() {

View File

@@ -119,7 +119,6 @@ func (arp *fsysARPDB) Neighbors() (ns []Neighbor) {
// //
// IP address HW type Flags HW address Mask Device // IP address HW type Flags HW address Mask Device
// 192.168.11.98 0x1 0x2 5a:92:df:a9:7e:28 * wan // 192.168.11.98 0x1 0x2 5a:92:df:a9:7e:28 * wan
//
func parseArpAWrt(sc *bufio.Scanner, lenHint int) (ns []Neighbor) { func parseArpAWrt(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
if !sc.Scan() { if !sc.Scan() {
// Skip the header. // Skip the header.
@@ -162,7 +161,6 @@ func parseArpAWrt(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
// expected input format: // expected input format:
// //
// hostname (192.168.1.1) at ab:cd:ef:ab:cd:ef [ether] on enp0s3 // hostname (192.168.1.1) at ab:cd:ef:ab:cd:ef [ether] on enp0s3
//
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) { func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint) ns = make([]Neighbor, 0, lenHint)
for sc.Scan() { for sc.Scan() {
@@ -209,7 +207,6 @@ func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
// expected input format: // expected input format:
// //
// 192.168.1.1 dev enp0s3 lladdr ab:cd:ef:ab:cd:ef REACHABLE // 192.168.1.1 dev enp0s3 lladdr ab:cd:ef:ab:cd:ef REACHABLE
//
func parseIPNeigh(sc *bufio.Scanner, lenHint int) (ns []Neighbor) { func parseIPNeigh(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint) ns = make([]Neighbor, 0, lenHint)
for sc.Scan() { for sc.Scan() {

View File

@@ -34,7 +34,6 @@ func newARPDB() (arp *cmdARPDB) {
// //
// Host Ethernet Address Netif Expire Flags // Host Ethernet Address Netif Expire Flags
// 192.168.1.1 ab:cd:ef:ab:cd:ef em0 19m59s // 192.168.1.1 ab:cd:ef:ab:cd:ef em0 19m59s
//
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) { func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
// Skip the header. // Skip the header.
if !sc.Scan() { if !sc.Scan() {

View File

@@ -25,12 +25,10 @@ func newARPDB() (arp *cmdARPDB) {
// parseArpA parses the output of the "arp /a" command on Windows. The expected // parseArpA parses the output of the "arp /a" command on Windows. The expected
// input format (the first line is empty): // input format (the first line is empty):
// //
//
// Interface: 192.168.56.16 --- 0x7 // Interface: 192.168.56.16 --- 0x7
// Internet Address Physical Address Type // Internet Address Physical Address Type
// 192.168.56.1 0a-00-27-00-00-00 dynamic // 192.168.56.1 0a-00-27-00-00-00 dynamic
// 192.168.56.255 ff-ff-ff-ff-ff-ff static // 192.168.56.255 ff-ff-ff-ff-ff-ff static
//
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) { func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint) ns = make([]Neighbor, 0, lenHint)
for sc.Scan() { for sc.Scan() {

View File

@@ -71,7 +71,6 @@ func (rm *requestMatcher) MatchRequest(
// a processed format like: // a processed format like:
// //
// ip host1 host2 ... // ip host1 host2 ...
//
func (rm *requestMatcher) Translate(rule string) (hostRule string) { func (rm *requestMatcher) Translate(rule string) (hostRule string) {
rm.stateLock.RLock() rm.stateLock.RLock()
defer rm.stateLock.RUnlock() defer rm.stateLock.RUnlock()

View File

@@ -64,7 +64,6 @@ func writeExit(w io.WriteCloser) {
// //
// Default Server: 192-168-1-1.qualified.domain.ru // Default Server: 192-168-1-1.qualified.domain.ru
// Address: 192.168.1.1 // Address: 192.168.1.1
//
func scanAddrs(s *bufio.Scanner) (addrs []string) { func scanAddrs(s *bufio.Scanner) (addrs []string) {
for s.Scan() { for s.Scan() {
line := strings.TrimSpace(s.Text()) line := strings.TrimSpace(s.Text())

View File

@@ -121,13 +121,12 @@ func PIDByCommand(command string, except ...int) (pid int, err error) {
} }
// parsePSOutput scans the output of ps searching the largest PID of the process // parsePSOutput scans the output of ps searching the largest PID of the process
// associated with cmdName ignoring PIDs from ignore. A valid line from // associated with cmdName ignoring PIDs from ignore. A valid line from r
// r should look like these: // should look like these:
// //
// 123 ./example-cmd // 123 ./example-cmd
// 1230 some/base/path/example-cmd // 1230 some/base/path/example-cmd
// 3210 example-cmd // 3210 example-cmd
//
func parsePSOutput(r io.Reader, cmdName string, ignore []int) (largest, instNum int, err error) { func parsePSOutput(r io.Reader, cmdName string, ignore []int) (largest, instNum int, err error) {
s := bufio.NewScanner(r) s := bufio.NewScanner(r)
for s.Scan() { for s.Scan() {

View File

@@ -65,39 +65,42 @@ func hwAddrToLinkLayerAddr(hwa net.HardwareAddr) (lla []byte, err error) {
} }
// Create an ICMPv6.RouterAdvertisement packet with all necessary options. // Create an ICMPv6.RouterAdvertisement packet with all necessary options.
// Data scheme:
// //
// ICMPv6: // ICMPv6:
// type[1] // - type[1]
// code[1] // - code[1]
// chksum[2] // - chksum[2]
// body (RouterAdvertisement): // - body (RouterAdvertisement):
// Cur Hop Limit[1] // - Cur Hop Limit[1]
// Flags[1]: MO...... // - Flags[1]: MO......
// Router Lifetime[2] // - Router Lifetime[2]
// Reachable Time[4] // - Reachable Time[4]
// Retrans Timer[4] // - Retrans Timer[4]
// Option=Prefix Information(3): // - Option=Prefix Information(3):
// Type[1] // - Type[1]
// Length * 8bytes[1] // - Length * 8bytes[1]
// Prefix Length[1] // - Prefix Length[1]
// Flags[1]: LA...... // - Flags[1]: LA......
// Valid Lifetime[4] // - Valid Lifetime[4]
// Preferred Lifetime[4] // - Preferred Lifetime[4]
// Reserved[4] // - Reserved[4]
// Prefix[16] // - Prefix[16]
// Option=MTU(5): // - Option=MTU(5):
// Type[1] // - Type[1]
// Length * 8bytes[1] // - Length * 8bytes[1]
// Reserved[2] // - Reserved[2]
// MTU[4] // - MTU[4]
// Option=Source link-layer address(1): // - Option=Source link-layer address(1):
// Link-Layer Address[8/24] // - Link-Layer Address[8/24]
// Option=Recursive DNS Server(25): // - Option=Recursive DNS Server(25):
// Type[1] // - Type[1]
// Length * 8bytes[1] // - Length * 8bytes[1]
// Reserved[2] // - Reserved[2]
// Lifetime[4] // - Lifetime[4]
// Addresses of IPv6 Recursive DNS Servers[16] // - Addresses of IPv6 Recursive DNS Servers[16]
//
// TODO(a.garipov): Replace with an existing implementation from a dependency.
func createICMPv6RAPacket(params icmpv6RA) (data []byte, err error) { func createICMPv6RAPacket(params icmpv6RA) (data []byte, err error) {
var lla []byte var lla []byte
lla, err = hwAddrToLinkLayerAddr(params.sourceLinkLayerAddress) lla, err = hwAddrToLinkLayerAddr(params.sourceLinkLayerAddress)

View File

@@ -267,6 +267,7 @@ func (f *Filtering) periodicallyRefreshFilters() {
// Refresh filters // Refresh filters
// flags: filterRefresh* // flags: filterRefresh*
// important: // important:
//
// TRUE: ignore the fact that we're currently updating the filters // TRUE: ignore the fact that we're currently updating the filters
func (f *Filtering) refreshFilters(flags int, important bool) (int, error) { func (f *Filtering) refreshFilters(flags int, important bool) (int, error) {
set := atomic.CompareAndSwapUint32(&f.refreshStatus, 0, 1) set := atomic.CompareAndSwapUint32(&f.refreshStatus, 0, 1)
@@ -363,25 +364,24 @@ const (
filterRefreshBlocklists = 4 // update block-lists filterRefreshBlocklists = 4 // update block-lists
) )
// Checks filters updates if necessary // refreshFiltersIfNecessary checks filters and updates them if necessary. If
// If force is true, it ignores the filter.LastUpdated field value // force is true, it ignores the filter.LastUpdated field value.
// flags: filterRefresh*
// //
// Algorithm: // Algorithm:
// . Get the list of filters to be updated
// . For each filter run the download and checksum check operation
// . Store downloaded data in a temporary file inside data/filters directory
// . For each filter:
// . If filter data hasn't changed, just set new update time on file
// . If filter data has changed:
// . rename the temporary file (<temp> -> 1.txt)
// Note that this method works only on UNIX.
// On Windows we don't pass files to filtering - we pass the whole data.
// . Pass new filters to filtering object - it analyzes new data while the old filters are still active
// . filtering activates new filters
// //
// Return the number of updated filters // 1. Get the list of filters to be updated. For each filter, run the download
// Return TRUE - there was a network error and nothing could be updated // and checksum check operation. Store downloaded data in a temporary file
// inside data/filters directory
//
// 2. For each filter, if filter data hasn't changed, just set new update time
// on file. Otherwise, rename the temporary file (<temp> -> 1.txt). Note
// that this method works only on Unix systems. On Windows, don't pass
// files to filtering, pass the whole data.
//
// refreshFiltersIfNecessary returns the number of updated filters. It also
// returns true if there was a network error and nothing could be updated.
//
// TODO(a.garipov, e.burkov): What the hell?
func (f *Filtering) refreshFiltersIfNecessary(flags int) (int, bool) { func (f *Filtering) refreshFiltersIfNecessary(flags int) (int, bool) {
log.Debug("Filters: updating...") log.Debug("Filters: updating...")

View File

@@ -741,11 +741,10 @@ func loadOptions() options {
// printWebAddrs prints addresses built from proto, addr, and an appropriate // printWebAddrs prints addresses built from proto, addr, and an appropriate
// port. At least one address is printed with the value of port. If the value // port. At least one address is printed with the value of port. If the value
// of betaPort is 0, the second address is not printed. The output example: // of betaPort is 0, the second address is not printed. Output example:
// //
// Go to http://127.0.0.1:80 // Go to http://127.0.0.1:80
// Go to http://127.0.0.1:3000 (BETA) // Go to http://127.0.0.1:3000 (BETA)
//
func printWebAddrs(proto, addr string, port, betaPort int) { func printWebAddrs(proto, addr string, port, betaPort int) {
const ( const (
hostMsg = "Go to %s://%s" hostMsg = "Go to %s://%s"

View File

@@ -159,13 +159,14 @@ func sendSigReload() {
} }
// handleServiceControlAction one of the possible control actions: // handleServiceControlAction one of the possible control actions:
// install -- installs a service/daemon //
// uninstall -- uninstalls it // - install: Installs a service/daemon.
// status -- prints the service status // - uninstall: Uninstalls it.
// start -- starts the previously installed service // - status: Prints the service status.
// stop -- stops the previously installed service // - start: Starts the previously installed service.
// restart - restarts the previously installed service // - stop: Stops the previously installed service.
// run - this is a special command that is not supposed to be used directly // - restart: Restarts the previously installed service.
// - run: This is a special command that is not supposed to be used directly
// it is specified when we register a service, and it indicates to the app // it is specified when we register a service, and it indicates to the app
// that it is being run as a service/daemon. // that it is being run as a service/daemon.
func handleServiceControlAction(opts options, clientBuildFS fs.FS) { func handleServiceControlAction(opts options, clientBuildFS fs.FS) {
@@ -402,7 +403,6 @@ var launchdConfig = `<?xml version='1.0' encoding='UTF-8'?>
// //
// 2. The ExecStartPre setting is added to make sure that the log directory is // 2. The ExecStartPre setting is added to make sure that the log directory is
// always created to prevent the 209/STDOUT errors. // always created to prevent the 209/STDOUT errors.
//
const systemdScript = `[Unit] const systemdScript = `[Unit]
Description={{.Description}} Description={{.Description}}
ConditionFileIsExecutable={{.Path|cmdEscape}} ConditionFileIsExecutable={{.Path|cmdEscape}}

View File

@@ -568,10 +568,9 @@ func validatePkey(data *tlsConfigStatus, pkey string) error {
return nil return nil
} }
// Process certificate data and its private key. // validateCertificates processes certificate data and its private key. All
// All parameters are optional. // parameters are optional. On error, validateCertificates returns a partially
// On error, return partially set object // set object with field WarningValidation containing error description.
// with 'WarningValidation' field containing error description.
func validateCertificates(certChain, pkey, serverName string) tlsConfigStatus { func validateCertificates(certChain, pkey, serverName string) tlsConfigStatus {
var data tlsConfigStatus var data tlsConfigStatus

View File

@@ -242,6 +242,7 @@ func upgradeSchema3to4(diskConf yobj) error {
// users: // users:
// - name: "..." // - name: "..."
// password: "..." // password: "..."
//
// ... // ...
func upgradeSchema4to5(diskConf yobj) error { func upgradeSchema4to5(diskConf yobj) error {
log.Printf("%s(): called", funcName()) log.Printf("%s(): called", funcName())
@@ -288,6 +289,7 @@ func upgradeSchema4to5(diskConf yobj) error {
// clients: // clients:
// ... // ...
//
// ip: 127.0.0.1 // ip: 127.0.0.1
// mac: ... // mac: ...
// //
@@ -295,6 +297,7 @@ func upgradeSchema4to5(diskConf yobj) error {
// //
// clients: // clients:
// ... // ...
//
// ids: // ids:
// - 127.0.0.1 // - 127.0.0.1
// - ... // - ...
@@ -355,6 +358,7 @@ func upgradeSchema5to6(diskConf yobj) error {
} }
// dhcp: // dhcp:
//
// enabled: false // enabled: false
// interface_name: vboxnet0 // interface_name: vboxnet0
// gateway_ip: 192.168.56.1 // gateway_ip: 192.168.56.1
@@ -363,6 +367,7 @@ func upgradeSchema5to6(diskConf yobj) error {
// -> // ->
// //
// dhcp: // dhcp:
//
// enabled: false // enabled: false
// interface_name: vboxnet0 // interface_name: vboxnet0
// dhcpv4: // dhcpv4:
@@ -451,7 +456,6 @@ func upgradeSchema6to7(diskConf yobj) error {
// 'dns': // 'dns':
// 'bind_hosts': // 'bind_hosts':
// - '127.0.0.1' // - '127.0.0.1'
//
func upgradeSchema7to8(diskConf yobj) (err error) { func upgradeSchema7to8(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 7 to 8") log.Printf("Upgrade yaml: 7 to 8")
@@ -488,7 +492,6 @@ func upgradeSchema7to8(diskConf yobj) (err error) {
// # AFTER: // # AFTER:
// 'dns': // 'dns':
// 'local_domain_name': 'lan' // 'local_domain_name': 'lan'
//
func upgradeSchema8to9(diskConf yobj) (err error) { func upgradeSchema8to9(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 8 to 9") log.Printf("Upgrade yaml: 8 to 9")
@@ -573,7 +576,6 @@ func addQUICPort(ups string, port int) (withPort string) {
// 'dns': // 'dns':
// 'upstream_dns': // 'upstream_dns':
// - 'quic://some-upstream.com:784' // - 'quic://some-upstream.com:784'
//
func upgradeSchema9to10(diskConf yobj) (err error) { func upgradeSchema9to10(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 9 to 10") log.Printf("Upgrade yaml: 9 to 10")
@@ -631,7 +633,6 @@ func upgradeSchema9to10(diskConf yobj) (err error) {
// 'group': '' // 'group': ''
// 'rlimit_nofile': 42 // 'rlimit_nofile': 42
// 'user': '' // 'user': ''
//
func upgradeSchema10to11(diskConf yobj) (err error) { func upgradeSchema10to11(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 10 to 11") log.Printf("Upgrade yaml: 10 to 11")
@@ -663,7 +664,6 @@ func upgradeSchema10to11(diskConf yobj) (err error) {
// //
// # AFTER: // # AFTER:
// 'querylog_interval': '2160h' // 'querylog_interval': '2160h'
//
func upgradeSchema11to12(diskConf yobj) (err error) { func upgradeSchema11to12(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 11 to 12") log.Printf("Upgrade yaml: 11 to 12")
diskConf["schema_version"] = 12 diskConf["schema_version"] = 12
@@ -707,7 +707,6 @@ func upgradeSchema11to12(diskConf yobj) (err error) {
// 'dhcp': // 'dhcp':
// # … // # …
// 'local_domain_name': 'lan' // 'local_domain_name': 'lan'
//
func upgradeSchema12to13(diskConf yobj) (err error) { func upgradeSchema12to13(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 12 to 13") log.Printf("Upgrade yaml: 12 to 13")
diskConf["schema_version"] = 13 diskConf["schema_version"] = 13
@@ -760,7 +759,6 @@ func upgradeSchema12to13(diskConf yobj) (err error) {
// 'rdns': true // 'rdns': true
// 'dhcp': true // 'dhcp': true
// 'hosts': true // 'hosts': true
//
func upgradeSchema13to14(diskConf yobj) (err error) { func upgradeSchema13to14(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 13 to 14") log.Printf("Upgrade yaml: 13 to 14")
diskConf["schema_version"] = 14 diskConf["schema_version"] = 14

View File

@@ -353,33 +353,25 @@ func topsCollector(units []*unitDB, max int, pg pairsGetter) []map[string]uint64
return convertTopSlice(a2) return convertTopSlice(a2)
} }
/* Algorithm: // getData returns the statistics data using the following algorithm:
. Prepare array of N units, where N is the value of "limit" configuration setting //
. Load data for the most recent units from file // 1. Prepare a slice of N units, where N is the value of "limit" configuration
If a unit with required ID doesn't exist, just add an empty unit // setting. Load data for the most recent units from the file. If a unit
. Get data for the current unit // with required ID doesn't exist, just add an empty unit. Get data for the
. Process data from the units and prepare an output map object: // current unit.
* per time unit counters: //
* DNS-queries/time-unit // 2. Process data from the units and prepare an output map object, including
* blocked/time-unit // per time unit counters (DNS queries per time-unit, blocked queries per
* safebrowsing-blocked/time-unit // time unit, etc.). If the time unit is hour, just add values from each
* parental-blocked/time-unit // unit to the slice; otherwise, the time unit is day, so aggregate per-hour
If time-unit is an hour, just add values from each unit to an array. // data into days.
If time-unit is a day, aggregate per-hour data into days. //
* top counters: // To get the top counters (queries per domain, queries per blocked domain,
* queries/domain // etc.), first sum up data for all units into a single map. Then, get the
* queries/blocked-domain // pairs with the highest numbers.
* queries/client //
To get these values we first sum up data for all units into a single map. // The total counters (DNS queries, blocked, etc.) are just the sum of data
Then we get the pairs with the highest numbers (the values are sorted in descending order) // for all units.
* total counters:
* DNS-queries
* blocked
* safebrowsing-blocked
* safesearch-blocked
* parental-blocked
These values are just the sum of data for all units.
*/
func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) { func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) {
if limit == 0 { if limit == 0 {
return StatsResp{ return StatsResp{