Pull request: 3435 openwrt detect

Merge in DNS/adguard-home from 3435-openwrt-detect to master Updates #3435. Squashed commit of the following: commit 04b10f407ced1c85ac8089f980d79e9bbfe14e95 Author: Eugene Burkov <e.burkov@adguard.com> Date: Fri Aug 13 19:02:55 2021 +0300 aghos: fix windows build commit d387cec5f9cae9256dccef8c666c02f2fb7449a2 Author: Eugene Burkov <e.burkov@adguard.com> Date: Fri Aug 13 18:22:12 2021 +0300 aghos: imp code, tests commit 2450b98522eb032ec8658f3ef2384fc77b627cc6 Author: Eugene Burkov <e.burkov@adguard.com> Date: Fri Aug 13 13:43:46 2021 +0300 all: imp code, docs commit 7fabba3a8dc70fe61dbaa8fd5445453816fe9ac7 Author: Eugene Burkov <e.burkov@adguard.com> Date: Fri Aug 13 04:04:09 2021 +0300 all: log changes commit 7cc1235308caf09eb4c80c05a4f328b8d6909ec7 Author: Eugene Burkov <e.burkov@adguard.com> Date: Fri Aug 13 03:33:13 2021 +0300 querylog: repl with golibs commit 84592087d3b2aca23613950bb203ff3c862624dc Author: Eugene Burkov <e.burkov@adguard.com> Date: Fri Aug 13 03:16:37 2021 +0300 aghos: use filewalker commit e4f2964b0e031c7a9a053e85c0ff7c792c772929 Author: Eugene Burkov <e.burkov@adguard.com> Date: Fri Aug 13 00:34:20 2021 +0300 aghos: mv recurrentchecker from aghnet
2021-08-13 19:20:17 +03:00
parent e3ad46876f
commit 394c2f65e0
16 changed files with 478 additions and 485 deletions
--- a/internal/aghos/sysutil_test.go
+++ b/internal/aghos/sysutil_test.go
--- a/internal/aghos/filewalker.go
+++ b/internal/aghos/filewalker.go
@@ -0,0 +1,119 @@
+package aghos
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/stringutil"
+)
+
+// FileWalker is the signature of a function called for files in the file tree.
+// As opposed to filepath.Walk it only walk the files (not directories) matching
+// the provided pattern and those returned by function itself.  All patterns
+// should be valid for filepath.Glob.  If cont is false, the walking terminates.
+// Each opened file is also limited for reading to MaxWalkedFileSize.
+//
+// TODO(e.burkov):  Consider moving to the separate package like pathutil.
+//
+// TODO(e.burkov):  Think about passing filename or any additional data.
+type FileWalker func(r io.Reader) (patterns []string, cont bool, err error)
+
+// MaxWalkedFileSize is the maximum length of the file that FileWalker can
+// check.
+const MaxWalkedFileSize = 1024 * 1024
+
+// checkFile tries to open and process a single file located on sourcePath.
+func checkFile(c FileWalker, sourcePath string) (patterns []string, cont bool, err error) {
+	var f *os.File
+	f, err = os.Open(sourcePath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			// Ignore non-existing files since this may only happen
+			// when the file was removed after filepath.Glob matched
+			// it.
+			return nil, true, nil
+		}
+
+		return nil, false, err
+	}
+	defer func() { err = errors.WithDeferred(err, f.Close()) }()
+
+	var r io.Reader
+	// Ignore the error since LimitReader function returns error only if
+	// passed limit value is less than zero, but the constant used.
+	//
+	// TODO(e.burkov):  Make variable.
+	r, _ = aghio.LimitReader(f, MaxWalkedFileSize)
+
+	return c(r)
+}
+
+// handlePatterns parses the patterns and ignores duplicates using srcSet.
+// srcSet must be non-nil.
+func handlePatterns(srcSet *stringutil.Set, patterns ...string) (sub []string, err error) {
+	sub = make([]string, 0, len(patterns))
+	for _, p := range patterns {
+		var matches []string
+		matches, err = filepath.Glob(p)
+		if err != nil {
+			// Enrich error with the pattern because filepath.Glob
+			// doesn't do it.
+			return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
+		}
+
+		for _, m := range matches {
+			if srcSet.Has(m) {
+				continue
+			}
+
+			srcSet.Add(m)
+			sub = append(sub, m)
+		}
+	}
+
+	return sub, nil
+}
+
+// Walk starts walking the files defined by initPattern.  It only returns true
+// if c signed to stop walking.
+func (c FileWalker) Walk(initPattern string) (ok bool, err error) {
+	// The slice of sources keeps the order in which the files are walked
+	// since srcSet.Values() returns strings in undefined order.
+	srcSet := stringutil.NewSet()
+	var src []string
+	src, err = handlePatterns(srcSet, initPattern)
+	if err != nil {
+		return false, err
+	}
+
+	var filename string
+	defer func() { err = errors.Annotate(err, "checking %q: %w", filename) }()
+
+	for i := 0; i < len(src); i++ {
+		var patterns []string
+		var cont bool
+		filename = src[i]
+		patterns, cont, err = checkFile(c, src[i])
+		if err != nil {
+			return false, err
+		}
+
+		if !cont {
+			return true, nil
+		}
+
+		var subsrc []string
+		subsrc, err = handlePatterns(srcSet, patterns...)
+		if err != nil {
+			return false, err
+		}
+
+		src = append(src, subsrc...)
+	}
+
+	return false, nil
+}
--- a/internal/aghos/filewalker_test.go
+++ b/internal/aghos/filewalker_test.go
@@ -0,0 +1,209 @@
+package aghos
+
+import (
+	"bufio"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testFSDir maps entries' names to entries which should either be a testFSDir
+// or byte slice.
+type testFSDir map[string]interface{}
+
+// testFSGen is used to generate a temporary filesystem consisting of
+// directories and plain text files from itself.
+type testFSGen testFSDir
+
+// gen returns the name of top directory of the generated filesystem.
+func (g testFSGen) gen(t *testing.T) (dirName string) {
+	t.Helper()
+
+	dirName = t.TempDir()
+	g.rangeThrough(t, dirName)
+
+	return dirName
+}
+
+func (g testFSGen) rangeThrough(t *testing.T, dirName string) {
+	const perm fs.FileMode = 0o777
+
+	for k, e := range g {
+		switch e := e.(type) {
+		case []byte:
+			require.NoError(t, os.WriteFile(filepath.Join(dirName, k), e, perm))
+
+		case testFSDir:
+			newDir := filepath.Join(dirName, k)
+			require.NoError(t, os.Mkdir(newDir, perm))
+
+			testFSGen(e).rangeThrough(t, newDir)
+		default:
+			t.Fatalf("unexpected entry type %T", e)
+		}
+	}
+}
+
+func TestFileWalker_Walk(t *testing.T) {
+	const attribute = `000`
+
+	makeFileWalker := func(dirName string) (fw FileWalker) {
+		return func(r io.Reader) (patterns []string, cont bool, err error) {
+			s := bufio.NewScanner(r)
+			for s.Scan() {
+				line := s.Text()
+				if line == attribute {
+					return nil, false, nil
+				}
+
+				if len(line) != 0 {
+					patterns = append(patterns, filepath.Join(dirName, line))
+				}
+			}
+
+			return patterns, true, s.Err()
+		}
+	}
+
+	const nl = "\n"
+
+	testCases := []struct {
+		name        string
+		testFS      testFSGen
+		initPattern string
+		want        bool
+	}{{
+		name: "simple",
+		testFS: testFSGen{
+			"simple_0001.txt": []byte(attribute + nl),
+		},
+		initPattern: "simple_0001.txt",
+		want:        true,
+	}, {
+		name: "chain",
+		testFS: testFSGen{
+			"chain_0001.txt": []byte(`chain_0002.txt` + nl),
+			"chain_0002.txt": []byte(`chain_0003.txt` + nl),
+			"chain_0003.txt": []byte(attribute + nl),
+		},
+		initPattern: "chain_0001.txt",
+		want:        true,
+	}, {
+		name: "several",
+		testFS: testFSGen{
+			"several_0001.txt": []byte(`several_*` + nl),
+			"several_0002.txt": []byte(`several_0001.txt` + nl),
+			"several_0003.txt": []byte(attribute + nl),
+		},
+		initPattern: "several_0001.txt",
+		want:        true,
+	}, {
+		name: "no",
+		testFS: testFSGen{
+			"no_0001.txt": []byte(nl),
+			"no_0002.txt": []byte(nl),
+			"no_0003.txt": []byte(nl),
+		},
+		initPattern: "no_*",
+		want:        false,
+	}, {
+		name: "subdirectory",
+		testFS: testFSGen{
+			"dir": testFSDir{
+				"subdir_0002.txt": []byte(attribute + nl),
+			},
+			"subdir_0001.txt": []byte(`dir/*`),
+		},
+		initPattern: "subdir_0001.txt",
+		want:        true,
+	}}
+
+	for _, tc := range testCases {
+		testDir := tc.testFS.gen(t)
+		fw := makeFileWalker(testDir)
+
+		t.Run(tc.name, func(t *testing.T) {
+			ok, err := fw.Walk(filepath.Join(testDir, tc.initPattern))
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.want, ok)
+		})
+	}
+
+	t.Run("pattern_malformed", func(t *testing.T) {
+		ok, err := makeFileWalker("").Walk("[]")
+		require.Error(t, err)
+
+		assert.False(t, ok)
+		assert.ErrorIs(t, err, filepath.ErrBadPattern)
+	})
+
+	t.Run("bad_filename", func(t *testing.T) {
+		dir := testFSGen{
+			"bad_filename.txt": []byte("[]"),
+		}.gen(t)
+		fw := FileWalker(func(r io.Reader) (patterns []string, cont bool, err error) {
+			s := bufio.NewScanner(r)
+			for s.Scan() {
+				patterns = append(patterns, s.Text())
+			}
+
+			return patterns, true, s.Err()
+		})
+
+		ok, err := fw.Walk(filepath.Join(dir, "bad_filename.txt"))
+		require.Error(t, err)
+
+		assert.False(t, ok)
+		assert.ErrorIs(t, err, filepath.ErrBadPattern)
+	})
+
+	t.Run("itself_error", func(t *testing.T) {
+		const rerr errors.Error = "returned error"
+
+		dir := testFSGen{
+			"mockfile.txt": []byte(`mockdata`),
+		}.gen(t)
+
+		ok, err := FileWalker(func(r io.Reader) (patterns []string, ok bool, err error) {
+			return nil, true, rerr
+		}).Walk(filepath.Join(dir, "*"))
+		require.Error(t, err)
+		require.False(t, ok)
+
+		assert.ErrorIs(t, err, rerr)
+	})
+}
+
+func TestWalkerFunc_CheckFile(t *testing.T) {
+	t.Run("non-existing", func(t *testing.T) {
+		_, ok, err := checkFile(nil, "lol")
+		require.NoError(t, err)
+
+		assert.True(t, ok)
+	})
+
+	t.Run("invalid_argument", func(t *testing.T) {
+		const badPath = "\x00"
+
+		_, ok, err := checkFile(nil, badPath)
+		require.Error(t, err)
+
+		assert.False(t, ok)
+		// TODO(e.burkov):  Use assert.ErrorsIs within the error from
+		// less platform-dependent package instead of syscall.EINVAL.
+		//
+		// See https://github.com/golang/go/issues/46849 and
+		// https://github.com/golang/go/issues/30322.
+		pathErr := &os.PathError{}
+		require.ErrorAs(t, err, &pathErr)
+		assert.Equal(t, "open", pathErr.Op)
+		assert.Equal(t, badPath, pathErr.Path)
+	})
+}
--- a/internal/aghos/os_linux.go
+++ b/internal/aghos/os_linux.go
@@ -4,11 +4,11 @@
 package aghos

 import (
-	"bytes"
+	"io"
 	"os"
-	"path/filepath"
-	"strings"
 	"syscall"
+
+	"github.com/AdguardTeam/golibs/stringutil"
 )

 func setRlimit(val uint64) (err error) {
@@ -30,37 +30,20 @@ func sendProcessSignal(pid int, sig syscall.Signal) error {
 }

 func isOpenWrt() (ok bool) {
-	const etcDir = "/etc"
+	var err error
+	ok, err = FileWalker(func(r io.Reader) (_ []string, cont bool, err error) {
+		const osNameData = "openwrt"

-	dirEnts, err := os.ReadDir(etcDir)
-	if err != nil {
-		return false
-	}
-
-	// fNameSubstr is a part of a name of the desired file.
-	const fNameSubstr = "release"
-	osNameData := []byte("OpenWrt")
-
-	for _, dirEnt := range dirEnts {
-		if dirEnt.IsDir() {
-			continue
-		}
-
-		fn := dirEnt.Name()
-		if !strings.Contains(fn, fNameSubstr) {
-			continue
-		}
-
-		var body []byte
-		body, err = os.ReadFile(filepath.Join(etcDir, fn))
+		// This use of ReadAll is now safe, because FileWalker's Walk()
+		// have limited r.
+		var data []byte
+		data, err = io.ReadAll(r)
 		if err != nil {
-			continue
+			return nil, false, err
 		}

-		if bytes.Contains(body, osNameData) {
-			return true
-		}
-	}
+		return nil, !stringutil.ContainsFold(string(data), osNameData), nil
+	}).Walk("/etc/*release*")

-	return false
+	return err == nil && ok
 }