Merge: * whois: improvements

Close #1035, Close #1036, Close #1047

* commit 'bd29b22f17d5b326d9fd603aa4f74593a0f1fca7':
  * CI: disable check if Git repo contains uncommited changes
  - whois: use the first "descr" field, not the last
  * filter: speed up parsing
  * whois: use whois.arin.net
  + whois, rdns: begin getting info for the most active clients on startup
  + stats: GetTopData()
  * stats: refactor

home/dns.go

@@ -60,6 +60,19 @@ func initDNSServer() {
 
 	config.dnsctx.rdns = InitRDNS(&config.clients)
 	config.dnsctx.whois = initWhois(&config.clients)
+
+	const topClientsNumber = 30 // the number of clients to get
+	topClients := config.stats.GetTopClientsIP(topClientsNumber)
+	for _, ip := range topClients {
+		ipAddr := net.ParseIP(ip)
+		if !ipAddr.IsLoopback() {
+			config.dnsctx.rdns.Begin(ip)
+		}
+		if isPublicIP(ipAddr) {
+			config.dnsctx.whois.Begin(ip)
+		}
+	}
+
 	initFiltering()
 }
 

home/filter.go

@@ -312,15 +312,14 @@ func isPrintableText(data []byte) bool {
 
 // A helper function that parses filter contents and returns a number of rules and a filter name (if there's any)
 func parseFilterContents(contents []byte) (int, string) {
-	lines := strings.Split(string(contents), "\n")
+	data := string(contents)
 	rulesCount := 0
 	name := ""
 	seenTitle := false
 
 	// Count lines in the filter
-	for _, line := range lines {
-
-		line = strings.TrimSpace(line)
+	for len(data) != 0 {
+		line := SplitNext(&data, '\n')
 		if len(line) == 0 {
 			continue
 		}

home/helpers.go

@@ -377,3 +377,18 @@ func parseIPv4(s string) net.IP {
 
 	return ip.To4()
 }
+
+// SplitNext - split string by a byte and return the first chunk
+// Whitespace is trimmed
+func SplitNext(str *string, splitBy byte) string {
+	i := strings.IndexByte(*str, splitBy)
+	s := ""
+	if i != -1 {
+		s = (*str)[0:i]
+		*str = (*str)[i+1:]
+	} else {
+		s = *str
+		*str = ""
+	}
+	return strings.TrimSpace(s)
+}

home/helpers_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGetValidNetInterfacesForWeb(t *testing.T) {
@@ -23,3 +24,10 @@ func TestGetValidNetInterfacesForWeb(t *testing.T) {
 		log.Printf("%v", iface)
 	}
 }
+
+func TestSplitNext(t *testing.T) {
+	s := " a,b , c "
+	assert.True(t, SplitNext(&s, ',') == "a")
+	assert.True(t, SplitNext(&s, ',') == "b")
+	assert.True(t, SplitNext(&s, ',') == "c" && len(s) == 0)
+}

home/whois.go

@@ -1,26 +1,35 @@
 package home
 
 import (
+	"fmt"
+	"io/ioutil"
+	"net"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/AdguardTeam/golibs/log"
-	whois "github.com/likexian/whois-go"
 )
 
-const maxValueLength = 250
+const (
+	defaultServer  = "whois.arin.net"
+	defaultPort    = "43"
+	maxValueLength = 250
+)
 
 // Whois - module context
 type Whois struct {
-	clients *clientsContainer
-	ips     map[string]bool
-	lock    sync.Mutex
-	ipChan  chan string
+	clients     *clientsContainer
+	ips         map[string]bool
+	lock        sync.Mutex
+	ipChan      chan string
+	timeoutMsec uint
 }
 
 // Create module context
 func initWhois(clients *clientsContainer) *Whois {
 	w := Whois{}
+	w.timeoutMsec = 5000
 	w.clients = clients
 	w.ips = make(map[string]bool)
 	w.ipChan = make(chan string, 255)
@@ -41,11 +50,9 @@ func whoisParse(data string) map[string]string {
 	m := map[string]string{}
 	descr := ""
 	netname := ""
-	lines := strings.Split(data, "\n")
-	for _, ln := range lines {
-		ln = strings.TrimSpace(ln)
-
-		if len(ln) == 0 || ln[0] == '#' {
+	for len(data) != 0 {
+		ln := SplitNext(&data, '\n')
+		if len(ln) == 0 || ln[0] == '#' || ln[0] == '%' {
 			continue
 		}
 
@@ -68,9 +75,19 @@ func whoisParse(data string) map[string]string {
 			m[k] = trimValue(v)
 
 		case "descr":
-			descr = v
+			if len(descr) == 0 {
+				descr = v
+			}
 		case "netname":
 			netname = v
+
+		case "whois": // "whois: whois.arin.net"
+			m["whois"] = v
+
+		case "referralserver": // "ReferralServer:  whois://whois.ripe.net"
+			if strings.HasPrefix(v, "whois://") {
+				m["whois"] = v[len("whois://"):]
+			}
 		}
 	}
 
@@ -85,10 +102,66 @@ func whoisParse(data string) map[string]string {
 	return m
 }
 
+// Send request to a server and receive the response
+func (w *Whois) query(target string, serverAddr string) (string, error) {
+	addr, _, _ := net.SplitHostPort(serverAddr)
+	if addr == "whois.arin.net" {
+		target = "n + " + target
+	}
+	conn, err := net.DialTimeout("tcp", serverAddr, time.Duration(w.timeoutMsec)*time.Millisecond)
+	if err != nil {
+		return "", err
+	}
+	defer conn.Close()
+
+	_ = conn.SetReadDeadline(time.Now().Add(time.Duration(w.timeoutMsec) * time.Millisecond))
+	_, err = conn.Write([]byte(target + "\r\n"))
+	if err != nil {
+		return "", err
+	}
+
+	data, err := ioutil.ReadAll(conn)
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
+}
+
+// Query WHOIS servers (handle redirects)
+func (w *Whois) queryAll(target string) (string, error) {
+	server := net.JoinHostPort(defaultServer, defaultPort)
+	const maxRedirects = 5
+	for i := 0; i != maxRedirects; i++ {
+		resp, err := w.query(target, server)
+		if err != nil {
+			return "", err
+		}
+		log.Debug("Whois: received response (%d bytes) from %s  IP:%s", len(resp), server, target)
+
+		m := whoisParse(resp)
+		redir, ok := m["whois"]
+		if !ok {
+			return resp, nil
+		}
+		redir = strings.ToLower(redir)
+
+		_, _, err = net.SplitHostPort(redir)
+		if err != nil {
+			server = net.JoinHostPort(redir, defaultPort)
+		} else {
+			server = redir
+		}
+
+		log.Debug("Whois: redirected to %s  IP:%s", redir, target)
+	}
+	return "", fmt.Errorf("Whois: redirect loop")
+}
+
 // Request WHOIS information
-func whoisProcess(ip string) [][]string {
+func (w *Whois) process(ip string) [][]string {
 	data := [][]string{}
-	resp, err := whois.Whois(ip)
+	resp, err := w.queryAll(ip)
 	if err != nil {
 		log.Debug("Whois: error: %s  IP:%s", err, ip)
 		return data
@@ -137,7 +210,7 @@ func (w *Whois) workerLoop() {
 		var ip string
 		ip = <-w.ipChan
 
-		info := whoisProcess(ip)
+		info := w.process(ip)
 		if len(info) == 0 {
 			continue
 		}

home/whois_test.go

@@ -1,19 +1,15 @@
 package home
 
 import (
-	"strings"
 	"testing"
 
-	whois "github.com/likexian/whois-go"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestWhois(t *testing.T) {
-	resp, err := whois.Whois("8.8.8.8")
+	w := Whois{timeoutMsec: 5000}
+	resp, err := w.queryAll("8.8.8.8")
 	assert.True(t, err == nil)
-	assert.True(t, strings.Index(resp, "OrgName:        Google LLC") != -1)
-	assert.True(t, strings.Index(resp, "City:           Mountain View") != -1)
-	assert.True(t, strings.Index(resp, "Country:        US") != -1)
 	m := whoisParse(resp)
 	assert.True(t, m["orgname"] == "Google LLC")
 	assert.True(t, m["country"] == "US")