all: sync with master

This commit is contained in:
Eugene Burkov
2024-12-05 16:00:18 +03:00
parent 54f3a5f990
commit 3f95db98d3
143 changed files with 3476 additions and 2959 deletions

View File

@@ -32,10 +32,10 @@ Required environment:
* `CHANNEL`: release channel, see above.
* `COMMIT`: current Git revision.
* `DIST_DIR`: the directory where a release has previously been built.
* `REVISION`: current Git revision.
* `VERSION`: release version.
Optional environment:
@@ -105,18 +105,6 @@ and call `make` with `GOTOOLCHAIN=local`.
### `clean.sh`: Cleanup
Optional environment:
* `GO`: set an alternative name for the Go compiler.
Required environment:
* `DIST_DIR`: the directory where a release has previously been built.
### `go-bench.sh`: Run backend benchmarks
Optional environment:

View File

@@ -16,6 +16,7 @@ import (
"text/template"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/logutil/slogutil"
)
@@ -30,14 +31,13 @@ func main() {
// Validate the URL.
_, err := url.Parse(urlStr)
check(err)
errors.Check(err)
c := &http.Client{
Timeout: 10 * time.Second,
}
resp, err := c.Get(urlStr)
check(err)
resp := errors.Must(c.Get(urlStr))
defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError)
if resp.StatusCode != http.StatusOK {
@@ -46,7 +46,7 @@ func main() {
hlSvcs := &hlServices{}
err = json.NewDecoder(resp.Body).Decode(hlSvcs)
check(err)
errors.Check(err)
// Sort all services and rules to make the output more predictable.
slices.SortStableFunc(hlSvcs.BlockedServices, func(a, b *hlServicesService) (res int) {
@@ -59,20 +59,20 @@ func main() {
// Use another set of delimiters to prevent them interfering with the Go
// code.
tmpl, err := template.New("main").Delims("<%", "%>").Funcs(template.FuncMap{
"isnotlast": func(idx, sliceLen int) (ok bool) { return idx != sliceLen-1 },
"isnotlast": func(idx, sliceLen int) (ok bool) {
return idx != sliceLen-1
},
}).Parse(tmplStr)
check(err)
errors.Check(err)
f, err := os.OpenFile(
f := errors.Must(os.OpenFile(
"./internal/filtering/servicelist.go",
os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
0o644,
)
check(err)
))
defer slogutil.CloseAndLog(ctx, l, f, slog.LevelError)
err = tmpl.Execute(f, hlSvcs)
check(err)
errors.Check(tmpl.Execute(f, hlSvcs))
}
// tmplStr is the template for the Go source file with the services.
@@ -100,13 +100,6 @@ var blockedServices = []blockedService{<% $l := len .BlockedServices %>
}<% if isnotlast $i $l %>, <% end %><% end %>}
`
// check is a simple error-checking helper for scripts.
func check(err error) {
if err != nil {
panic(err)
}
}
// hlServices is the JSON structure for the Hostlists Registry blocked service
// index.
type hlServices struct {

View File

@@ -2,81 +2,78 @@
set -e -f -u
# This comment is used to simplify checking local copies of the script.
# Bump this number every time a significant change is made to this
# script.
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 3
# AdGuard-Project-Version: 5
# TODO(a.garipov): Add pre-merge-commit.
# Only show interactive prompts if there a terminal is attached to
# stdout. While this technically doesn't guarantee that reading from
# /dev/tty works, this should work reasonably well on all of our
# supported development systems and in most terminal emulators.
# Only show interactive prompts if there a terminal is attached to stdout.
# While this technically doesn't guarantee that reading from /dev/tty works,
# this should work reasonably well on all of our supported development systems
# and in most terminal emulators.
is_tty='0'
if [ -t '1' ]
then
if [ -t '1' ]; then
is_tty='1'
fi
readonly is_tty
# prompt is a helper that prompts the user for interactive input if that
# can be done. If there is no terminal attached, it sleeps for two
# seconds, giving the programmer some time to react, and returns with
# a zero exit code.
# prompt is a helper that prompts the user for interactive input if that can be
# done. If there is no terminal attached, it sleeps for two seconds, giving the
# programmer some time to react, and returns with a zero exit code.
prompt() {
if [ "$is_tty" -eq '0' ]
then
if [ "$is_tty" -eq '0' ]; then
sleep 2
return 0
fi
while true
do
while true; do
printf 'commit anyway? y/[n]: '
read -r ans < /dev/tty
read -r ans </dev/tty
case "$ans"
in
('y'|'Y')
case "$ans" in
'y' | 'Y')
break
;;
(''|'n'|'N')
'' | 'n' | 'N')
exit 1
;;
(*)
*)
continue
;;
esac
done
}
# Warn the programmer about unstaged changes and untracked files, but do
# not fail the commit, because those changes might be temporary or for
# a different branch.
# Warn the programmer about unstaged changes and untracked files, but do not
# fail the commit, because those changes might be temporary or for a different
# branch.
#
# shellcheck disable=SC2016
awk_prog='substr($2, 2, 1) != "." { print $9; } $1 == "?" { print $2; }'
readonly awk_prog
unstaged="$( git status --porcelain=2 | awk "$awk_prog" )"
unstaged="$(git status --porcelain=2 | awk "$awk_prog")"
readonly unstaged
if [ "$unstaged" != '' ]
then
if [ "$unstaged" != '' ]; then
printf 'WARNING: you have unstaged changes:\n\n%s\n\n' "$unstaged"
prompt
fi
# Warn the programmer about temporary todos, but do not fail the commit,
# because the commit could be in a temporary branch.
temp_todos="$( git grep -e 'TODO.*!!' -- ':!scripts/hooks/pre-commit' || : )"
# Warn the programmer about temporary todos and skel FIXMEs, but do not fail the
# commit, because the commit could be in a temporary branch.
temp_todos="$(
git grep -e 'FIXME' -e 'TODO.*!!' -- \
':!./scripts/hooks/pre-commit' \
':!./client' \
|| :
)"
readonly temp_todos
if [ "$temp_todos" != '' ]
then
if [ "$temp_todos" != '' ]; then
printf 'WARNING: you have temporary todos:\n\n%s\n\n' "$temp_todos"
prompt
fi
@@ -84,32 +81,22 @@ fi
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$( git diff --cached --name-only -- 'client/*.js' || true )" != '' ]
then
make VERBOSE="$verbose" js-lint js-test
fi
if [ "$( git diff --cached --name-only -- '*.go' '*.mod' 'Makefile' || true )" != '' ]
then
make VERBOSE="$verbose" go-os-check go-lint go-test
fi
if [ "$( git diff --cached --name-only -- '*.md' || true )" != '' ]
then
if [ "$(git diff --cached --name-only -- '*.md' || :)" != '' ]; then
make VERBOSE="$verbose" md-lint
fi
if [ "$( git diff --cached --name-only -- '*.sh' || true )" != '' ]
then
if [ "$(git diff --cached --name-only -- '*.sh' || :)" != '' ]; then
make VERBOSE="$verbose" sh-lint
fi
if [ "$( git diff --cached --name-only -- '*.md' '*.txt' '*.yaml' '*.yml' || true )" != '' ]
then
if [ "$(git diff --cached --name-only -- '*.md' '*.txt' '*.yaml' '*.yml' || :)" != '' ]; then
make VERBOSE="$verbose" txt-lint
fi
if [ "$( git diff --cached --name-only -- './openapi/openapi.yaml' || true )" != '' ]
then
if [ "$(git diff --cached --name-only -- '*.go' '*.mod' 'Makefile' || :)" != '' ]; then
make VERBOSE="$verbose" go-os-check go-lint go-test
fi
if [ "$(git diff --cached --name-only -- './openapi/openapi.yaml' || :)" != '' ]; then
make VERBOSE="$verbose" openapi-lint
fi

View File

@@ -9,8 +9,7 @@ set -e -f -u
# Function log is an echo wrapper that writes to stderr if the caller
# requested verbosity level greater than 0. Otherwise, it does nothing.
log() {
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
echo "$1" 1>&2
fi
}
@@ -27,7 +26,7 @@ error_exit() {
#
# TODO(e.burkov): Document each option.
usage() {
echo 'install.sh: usage: [-c channel] [-C cpu_type] [-h] [-O os] [-o output_dir]'\
echo 'install.sh: usage: [-c channel] [-C cpu_type] [-h] [-O os] [-o output_dir]' \
'[-r|-R] [-u|-U] [-v|-V]' 1>&2
exit 2
@@ -38,8 +37,7 @@ usage() {
#
# TODO(e.burkov): Use everywhere the sudo_cmd isn't quoted.
maybe_sudo() {
if [ "$use_sudo" -eq 0 ]
then
if [ "$use_sudo" -eq 0 ]; then
"$@"
else
"$sudo_cmd" "$@"
@@ -64,8 +62,8 @@ is_little_endian() {
# explicitly implementation-defined in POSIX. Use hexdump instead of od,
# because OpenWrt and its derivatives have the former but not the latter.
is_little_endian_result="$(
printf 'I'\
| hexdump -o\
printf 'I' \
| hexdump -o \
| awk '{ print substr($2, 6, 1); exit; }'
)"
readonly is_little_endian_result
@@ -84,15 +82,14 @@ check_required() {
required_unix="tar"
readonly required_darwin required_unix
case "$os"
in
('freebsd'|'linux'|'openbsd')
case "$os" in
'freebsd' | 'linux' | 'openbsd')
required="$required_unix"
;;
('darwin')
'darwin')
required="$required_darwin"
;;
(*)
*)
# Generally shouldn't happen, since the OS has already been validated.
error_exit "unsupported operating system: '$os'"
;;
@@ -100,11 +97,9 @@ check_required() {
readonly required
# Don't use quotes to get word splitting.
for cmd in $required
do
for cmd in $required; do
log "checking $cmd"
if ! is_command "$cmd"
then
if ! is_command "$cmd"; then
log "the full list of required software: [$required]"
error_exit "$cmd is required to install AdGuard Home via this script"
@@ -114,57 +109,53 @@ check_required() {
# Function check_out_dir requires the output directory to be set and exist.
check_out_dir() {
if [ "$out_dir" = '' ]
then
if [ "$out_dir" = '' ]; then
error_exit 'output directory should be presented'
fi
if ! [ -d "$out_dir" ]
then
if ! [ -d "$out_dir" ]; then
log "$out_dir directory will be created"
fi
}
# Function parse_opts parses the options list and validates it's combinations.
parse_opts() {
while getopts "C:c:hO:o:rRuUvV" opt "$@"
do
case "$opt"
in
(C)
while getopts "C:c:hO:o:rRuUvV" opt "$@"; do
case "$opt" in
C)
cpu="$OPTARG"
;;
(c)
c)
channel="$OPTARG"
;;
(h)
h)
usage
;;
(O)
O)
os="$OPTARG"
;;
(o)
o)
out_dir="$OPTARG"
;;
(R)
R)
reinstall='0'
;;
(U)
U)
uninstall='0'
;;
(r)
r)
reinstall='1'
;;
(u)
u)
uninstall='1'
;;
(V)
V)
verbose='0'
;;
(v)
v)
verbose='1'
;;
(*)
*)
log "bad option $OPTARG"
usage
@@ -172,8 +163,7 @@ parse_opts() {
esac
done
if [ "$uninstall" -eq '1' ] && [ "$reinstall" -eq '1' ]
then
if [ "$uninstall" -eq '1' ] && [ "$reinstall" -eq '1' ]; then
error_exit 'the -r and -u options are mutually exclusive'
fi
}
@@ -181,14 +171,12 @@ parse_opts() {
# Function set_channel sets the channel if needed and validates the value.
set_channel() {
# Validate.
case "$channel"
in
('development'|'edge'|'beta'|'release')
case "$channel" in
'development' | 'edge' | 'beta' | 'release')
# All is well, go on.
;;
(*)
error_exit \
"invalid channel '$channel'
*)
error_exit "invalid channel '$channel'
supported values are 'development', 'edge', 'beta', and 'release'"
;;
esac
@@ -200,36 +188,33 @@ supported values are 'development', 'edge', 'beta', and 'release'"
# Function set_os sets the os if needed and validates the value.
set_os() {
# Set if needed.
if [ "$os" = '' ]
then
os="$( uname -s )"
case "$os"
in
('Darwin')
if [ "$os" = '' ]; then
os="$(uname -s)"
case "$os" in
'Darwin')
os='darwin'
;;
('FreeBSD')
'FreeBSD')
os='freebsd'
;;
('Linux')
'Linux')
os='linux'
;;
('OpenBSD')
'OpenBSD')
os='openbsd'
;;
(*)
*)
error_exit "unsupported operating system: '$os'"
;;
esac
fi
# Validate.
case "$os"
in
('darwin'|'freebsd'|'linux'|'openbsd')
case "$os" in
'darwin' | 'freebsd' | 'linux' | 'openbsd')
# All right, go on.
;;
(*)
*)
error_exit "unsupported operating system: '$os'"
;;
esac
@@ -241,52 +226,49 @@ set_os() {
# Function set_cpu sets the cpu if needed and validates the value.
set_cpu() {
# Set if needed.
if [ "$cpu" = '' ]
then
cpu="$( uname -m )"
case "$cpu"
in
('x86_64'|'x86-64'|'x64'|'amd64')
if [ "$cpu" = '' ]; then
cpu="$(uname -m)"
case "$cpu" in
'x86_64' | 'x86-64' | 'x64' | 'amd64')
cpu='amd64'
;;
('i386'|'i486'|'i686'|'i786'|'x86')
'i386' | 'i486' | 'i686' | 'i786' | 'x86')
cpu='386'
;;
('armv5l')
'armv5l')
cpu='armv5'
;;
('armv6l')
'armv6l')
cpu='armv6'
;;
('armv7l' | 'armv8l')
'armv7l' | 'armv8l')
cpu='armv7'
;;
('aarch64'|'arm64')
'aarch64' | 'arm64')
cpu='arm64'
;;
('mips'|'mips64')
if is_little_endian
then
'mips' | 'mips64')
if is_little_endian; then
cpu="${cpu}le"
fi
cpu="${cpu}_softfloat"
;;
(*)
*)
error_exit "unsupported cpu type: $cpu"
;;
esac
fi
# Validate.
case "$cpu"
in
('amd64'|'386'|'armv5'|'armv6'|'armv7'|'arm64')
case "$cpu" in
'amd64' | '386' | 'armv5' | 'armv6' | 'armv7' | 'arm64')
# All right, go on.
;;
('mips64le_softfloat'|'mips64_softfloat'|'mipsle_softfloat'|'mips_softfloat')
'mips64le_softfloat' | 'mips64_softfloat' | 'mipsle_softfloat' | 'mips_softfloat')
# That's right too.
;;
(*)
*)
error_exit "unsupported cpu type: $cpu"
;;
esac
@@ -301,8 +283,7 @@ set_cpu() {
#
# See https://github.com/AdguardTeam/AdGuardHome/issues/2443.
fix_darwin() {
if [ "$os" != 'darwin' ]
then
if [ "$os" != 'darwin' ]; then
return 0
fi
@@ -317,16 +298,14 @@ fix_darwin() {
# Function fix_freebsd performs some fixes to make it work on FreeBSD.
fix_freebsd() {
if ! [ "$os" = 'freebsd' ]
then
if ! [ "$os" = 'freebsd' ]; then
return 0
fi
rcd='/usr/local/etc/rc.d'
readonly rcd
if ! [ -d "$rcd" ]
then
if ! [ -d "$rcd" ]; then
mkdir "$rcd"
fi
}
@@ -335,8 +314,7 @@ fix_freebsd() {
# The second argument is optional and is the output file.
download_curl() {
curl_output="${2:-}"
if [ "$curl_output" = '' ]
then
if [ "$curl_output" = '' ]; then
curl -L -S -s "$1"
else
curl -L -S -o "$curl_output" -s "$1"
@@ -355,8 +333,7 @@ download_wget() {
# URL. The second argument is optional and is the output file.
download_fetch() {
fetch_output="${2:-}"
if [ "$fetch_output" = '' ]
then
if [ "$fetch_output" = '' ]; then
fetch -o '-' "$1"
else
fetch -o "$fetch_output" "$1"
@@ -366,15 +343,12 @@ download_fetch() {
# Function set_download_func sets the appropriate function for downloading
# files.
set_download_func() {
if is_command 'curl'
then
if is_command 'curl'; then
# Go on and use the default, download_curl.
return 0
elif is_command 'wget'
then
elif is_command 'wget'; then
download_func='download_wget'
elif is_command 'fetch'
then
elif is_command 'fetch'; then
download_func='download_fetch'
else
error_exit "either curl or wget is required to install AdGuard Home via this script"
@@ -384,15 +358,14 @@ set_download_func() {
# Function set_sudo_cmd sets the appropriate command to run a command under
# superuser privileges.
set_sudo_cmd() {
case "$os"
in
('openbsd')
case "$os" in
'openbsd')
sudo_cmd='doas'
;;
('darwin'|'freebsd'|'linux')
'darwin' | 'freebsd' | 'linux')
# Go on and use the default, sudo.
;;
(*)
*)
error_exit "unsupported operating system: '$os'"
;;
esac
@@ -418,22 +391,20 @@ configure() {
# Function is_root checks for root privileges to be granted.
is_root() {
if [ "$( id -u )" -eq '0' ]
then
user_id="$(id -u)"
if [ "$user_id" -eq '0' ]; then
log 'script is executed with root privileges'
return 0
fi
if is_command "$sudo_cmd"
then
if is_command "$sudo_cmd"; then
log 'note that AdGuard Home requires root privileges to install using this script'
return 1
fi
error_exit \
'root privileges are required to install AdGuard Home using this script
error_exit 'root privileges are required to install AdGuard Home using this script
please, restart it with root privileges'
}
@@ -443,25 +414,21 @@ please, restart it with root privileges'
#
# TODO(e.burkov): Try to avoid restarting.
rerun_with_root() {
script_url=\
'https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/scripts/install.sh'
script_url='https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/scripts/install.sh'
readonly script_url
r='-R'
if [ "$reinstall" -eq '1' ]
then
if [ "$reinstall" -eq '1' ]; then
r='-r'
fi
u='-U'
if [ "$uninstall" -eq '1' ]
then
if [ "$uninstall" -eq '1' ]; then
u='-u'
fi
v='-V'
if [ "$verbose" -eq '1' ]
then
if [ "$verbose" -eq '1' ]; then
v='-v'
fi
@@ -473,7 +440,7 @@ rerun_with_root() {
# producing any output, the latter prints an exit command for the following
# shell to execute to prevent it from getting an empty input and exiting
# with a zero code in that case.
{ "$download_func" "$script_url" || echo 'exit 1'; }\
{ "$download_func" "$script_url" || echo 'exit 1'; } \
| $sudo_cmd sh -s -- -c "$channel" -C "$cpu" -O "$os" -o "$out_dir" "$r" "$u" "$v"
# Exit the script. Since if the code of the previous pipeline is non-zero,
@@ -484,10 +451,9 @@ rerun_with_root() {
# Function download downloads the file from the URL and saves it to the
# specified filepath.
download() {
log "downloading package from $url -> $pkg_name"
log "downloading package from $url to $pkg_name"
if ! "$download_func" "$url" "$pkg_name"
then
if ! "$download_func" "$url" "$pkg_name"; then
error_exit "cannot download the package from $url into $pkg_name"
fi
@@ -497,25 +463,29 @@ download() {
# Function unpack unpacks the passed archive depending on it's extension.
unpack() {
log "unpacking package from $pkg_name into $out_dir"
if ! mkdir -m 0700 -p "$out_dir"
then
# shellcheck disable=SC2174
if ! mkdir -m 0700 -p "$out_dir"; then
error_exit "cannot create directory $out_dir"
fi
case "$pkg_ext"
in
('zip')
case "$pkg_ext" in
'zip')
unzip "$pkg_name" -d "$out_dir"
;;
('tar.gz')
'tar.gz')
tar -C "$out_dir" -f "$pkg_name" -x -z
;;
(*)
*)
error_exit "unexpected package extension: '$pkg_ext'"
;;
esac
log "successfully unpacked, contents: $( echo; ls -l -A "$agh_dir" )"
unpacked_contents="$(
echo
ls -l -A "$agh_dir"
)"
log "successfully unpacked, contents: $unpacked_contents"
rm "$pkg_name"
}
@@ -523,34 +493,29 @@ unpack() {
# Function handle_existing detects the existing AGH installation and takes care
# of removing it if needed.
handle_existing() {
if ! [ -d "$agh_dir" ]
then
if ! [ -d "$agh_dir" ]; then
log 'no need to uninstall'
if [ "$uninstall" -eq '1' ]
then
if [ "$uninstall" -eq '1' ]; then
exit 0
fi
return 0
fi
if [ "$( ls -1 -A "$agh_dir" )" != '' ]
then
existing_adguard_home="$(ls -1 -A "$agh_dir")"
if [ "$existing_adguard_home" != '' ]; then
log 'the existing AdGuard Home installation is detected'
if [ "$reinstall" -ne '1' ] && [ "$uninstall" -ne '1' ]
then
if [ "$reinstall" -ne '1' ] && [ "$uninstall" -ne '1' ]; then
error_exit \
"to reinstall/uninstall the AdGuard Home using this script specify one of the '-r' or '-u' flags"
"to reinstall/uninstall the AdGuard Home using this script specify one of the '-r' or '-u' flags"
fi
# TODO(e.burkov): Remove the stop once v0.107.1 released.
if ( cd "$agh_dir" && ! ./AdGuardHome -s stop || ! ./AdGuardHome -s uninstall )
then
# It doesn't terminate the script since it is possible
# that AGH just not installed as service but appearing
# in the directory.
if (cd "$agh_dir" && ! ./AdGuardHome -s stop || ! ./AdGuardHome -s uninstall); then
# It doesn't terminate the script since it is possible that AGH just
# not installed as service but appearing in the directory.
log "cannot uninstall AdGuard Home from $agh_dir"
fi
@@ -559,8 +524,7 @@ handle_existing() {
log 'AdGuard Home was successfully uninstalled'
fi
if [ "$uninstall" -eq '1' ]
then
if [ "$uninstall" -eq '1' ]; then
exit 0
fi
}
@@ -569,13 +533,11 @@ handle_existing() {
install_service() {
# Installing the service as root is required at least on FreeBSD.
use_sudo='0'
if [ "$os" = 'freebsd' ]
then
if [ "$os" = 'freebsd' ]; then
use_sudo='1'
fi
if ( cd "$agh_dir" && maybe_sudo ./AdGuardHome -s install )
then
if (cd "$agh_dir" && maybe_sudo ./AdGuardHome -s install); then
return 0
fi
@@ -583,13 +545,11 @@ install_service() {
rm -r "$agh_dir"
# Some devices detected to have armv7 CPU face the compatibility
# issues with actual armv7 builds. We should try to install the
# armv5 binary instead.
# Some devices detected to have armv7 CPU face the compatibility issues with
# actual armv7 builds. We should try to install the armv5 binary instead.
#
# See https://github.com/AdguardTeam/AdGuardHome/issues/2542.
if [ "$cpu" = 'armv7' ]
then
if [ "$cpu" = 'armv7' ]; then
cpu='armv5'
reinstall='1'
@@ -601,8 +561,6 @@ install_service() {
error_exit 'cannot install AdGuardHome as a service'
}
# Entrypoint
# Set default values of configuration variables.
@@ -624,8 +582,7 @@ echo 'starting AdGuard Home installation script'
configure
check_required
if ! is_root
then
if ! is_root; then
rerun_with_root
fi
# Needs rights.
@@ -638,7 +595,7 @@ unpack
install_service
echo "\
AdGuard Home is now installed and running
you can control the service status with the following commands:
$sudo_cmd ${agh_dir}/AdGuardHome -s start|stop|restart|status|install|uninstall"
printf '%s\n' \
'AdGuard Home is now installed and running' \
'you can control the service status with the following commands:' \
"$sudo_cmd ${agh_dir}/AdGuardHome -s start|stop|restart|status|install|uninstall"

View File

@@ -2,8 +2,7 @@
verbose="${VERBOSE:-0}"
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
debug_flags='--debug=1'
else
@@ -16,13 +15,12 @@ set -e -f -u
# Require these to be set. The channel value is validated later.
channel="${CHANNEL:?please set CHANNEL}"
commit="${COMMIT:?please set COMMIT}"
commit="${REVISION:?please set REVISION}"
dist_dir="${DIST_DIR:?please set DIST_DIR}"
readonly channel commit dist_dir
if [ "${VERSION:-}" = 'v0.0.0' ] || [ "${VERSION:-}" = '' ]
then
version="$( sh ./scripts/make/version.sh )"
if [ "${VERSION:-}" = 'v0.0.0' ] || [ "${VERSION:-}" = '' ]; then
version="$(sh ./scripts/make/version.sh)"
else
version="$VERSION"
fi
@@ -41,7 +39,7 @@ linux/arm64,\
linux/ppc64le"
readonly docker_platforms
build_date="$( date -u +'%Y-%m-%dT%H:%M:%SZ' )"
build_date="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
readonly build_date
# Set DOCKER_IMAGE_NAME to 'adguard/adguard-home' if you want (and are allowed)
@@ -59,27 +57,26 @@ readonly docker_image_name
docker_output="${DOCKER_OUTPUT:-type=image,name=${docker_image_name},push=false}"
readonly docker_output
case "$channel"
in
('release')
case "$channel" in
'release')
docker_version_tag="--tag=${docker_image_name}:${version}"
docker_channel_tag="--tag=${docker_image_name}:latest"
;;
('beta')
'beta')
docker_version_tag="--tag=${docker_image_name}:${version}"
docker_channel_tag="--tag=${docker_image_name}:beta"
;;
('edge')
'edge')
# Set the version tag to an empty string when pushing to the edge channel.
docker_version_tag=''
docker_channel_tag="--tag=${docker_image_name}:edge"
;;
('development')
'development')
# Set both tags to an empty string for development builds.
docker_version_tag=''
docker_channel_tag=''
;;
(*)
*)
echo "invalid channel '$channel', supported values are\
'development', 'edge', 'beta', and 'release'" 1>&2
exit 1
@@ -94,17 +91,17 @@ dist_docker="${dist_dir}/docker"
readonly dist_docker
mkdir -p "$dist_docker"
cp "${dist_dir}/AdGuardHome_linux_386/AdGuardHome/AdGuardHome"\
cp "${dist_dir}/AdGuardHome_linux_386/AdGuardHome/AdGuardHome" \
"${dist_docker}/AdGuardHome_linux_386_"
cp "${dist_dir}/AdGuardHome_linux_amd64/AdGuardHome/AdGuardHome"\
cp "${dist_dir}/AdGuardHome_linux_amd64/AdGuardHome/AdGuardHome" \
"${dist_docker}/AdGuardHome_linux_amd64_"
cp "${dist_dir}/AdGuardHome_linux_arm64/AdGuardHome/AdGuardHome"\
cp "${dist_dir}/AdGuardHome_linux_arm64/AdGuardHome/AdGuardHome" \
"${dist_docker}/AdGuardHome_linux_arm64_"
cp "${dist_dir}/AdGuardHome_linux_arm_6/AdGuardHome/AdGuardHome"\
cp "${dist_dir}/AdGuardHome_linux_arm_6/AdGuardHome/AdGuardHome" \
"${dist_docker}/AdGuardHome_linux_arm_v6"
cp "${dist_dir}/AdGuardHome_linux_arm_7/AdGuardHome/AdGuardHome"\
cp "${dist_dir}/AdGuardHome_linux_arm_7/AdGuardHome/AdGuardHome" \
"${dist_docker}/AdGuardHome_linux_arm_v7"
cp "${dist_dir}/AdGuardHome_linux_ppc64le/AdGuardHome/AdGuardHome"\
cp "${dist_dir}/AdGuardHome_linux_ppc64le/AdGuardHome/AdGuardHome" \
"${dist_docker}/AdGuardHome_linux_ppc64le_"
# Don't use quotes with $docker_version_tag and $docker_channel_tag, because we
@@ -112,16 +109,14 @@ cp "${dist_dir}/AdGuardHome_linux_ppc64le/AdGuardHome/AdGuardHome"\
#
# TODO(a.garipov): Once flag --tag of docker buildx build supports commas, use
# them instead.
$sudo_cmd docker\
"$debug_flags"\
buildx build\
--build-arg BUILD_DATE="$build_date"\
--build-arg DIST_DIR="$dist_dir"\
--build-arg VCS_REF="$commit"\
--build-arg VERSION="$version"\
--output "$docker_output"\
--platform "$docker_platforms"\
$docker_version_tag\
$docker_channel_tag\
-f ./docker/Dockerfile\
.
#
# shellcheck disable=SC2086
$sudo_cmd docker "$debug_flags" \
buildx build \
--build-arg BUILD_DATE="$build_date" \
--build-arg DIST_DIR="$dist_dir" \
--build-arg VCS_REF="$commit" \
--build-arg VERSION="$version" \
--output "$docker_output" \
--platform "$docker_platforms" \
$docker_version_tag $docker_channel_tag -f ./docker/Dockerfile .

View File

@@ -15,8 +15,7 @@
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
env
set -x
fi
@@ -32,8 +31,7 @@ set -e -f -u
# Function log is an echo wrapper that writes to stderr if the caller requested
# verbosity level greater than 0. Otherwise, it does nothing.
log() {
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
# Don't use quotes to get word splitting.
echo "$1" 1>&2
fi
@@ -49,9 +47,8 @@ readonly channel
# Check VERSION against the default value from the Makefile. If it is that, use
# the version calculation script.
version="${VERSION:-}"
if [ "$version" = 'v0.0.0' ] || [ "$version" = '' ]
then
version="$( sh ./scripts/make/version.sh )"
if [ "$version" = 'v0.0.0' ] || [ "$version" = '' ]; then
version="$(sh ./scripts/make/version.sh)"
fi
readonly version
@@ -60,8 +57,7 @@ log "version '$version'"
# Check architecture and OS limiters. Add spaces to the local versions for
# better pattern matching.
if [ "${ARCH:-}" != '' ]
then
if [ "${ARCH:-}" != '' ]; then
log "arches: '$ARCH'"
arches=" $ARCH "
else
@@ -69,8 +65,7 @@ else
fi
readonly arches
if [ "${OS:-}" != '' ]
then
if [ "${OS:-}" != '' ]; then
log "oses: '$OS'"
oses=" $OS "
else
@@ -79,8 +74,7 @@ fi
readonly oses
# Require the gpg key and passphrase to be set if the signing is required.
if [ "$sign" -eq '1' ]
then
if [ "$sign" -eq '1' ]; then
gpg_key_passphrase="${GPG_KEY_PASSPHRASE:?please set GPG_KEY_PASSPHRASE or unset SIGN}"
gpg_key="${GPG_KEY:?please set GPG_KEY or unset SIGN}"
signer_api_key="${SIGNER_API_KEY:?please set SIGNER_API_KEY or unset SIGN}"
@@ -102,12 +96,9 @@ log "checking tools"
# Make sure we fail gracefully if one of the tools we need is missing. Use
# alternatives when available.
use_shasum='0'
for tool in gpg gzip sed sha256sum tar zip
do
if ! command -v "$tool" > /dev/null
then
if [ "$tool" = 'sha256sum' ] && command -v 'shasum' > /dev/null
then
for tool in gpg gzip sed sha256sum tar zip; do
if ! command -v "$tool" >/dev/null; then
if [ "$tool" = 'sha256sum' ] && command -v 'shasum' >/dev/null; then
# macOS doesn't have sha256sum installed by default, but it does
# have shasum.
log 'replacing sha256sum with shasum -a 256'
@@ -157,8 +148,7 @@ readonly platforms
# system.
sign() {
# Only sign if needed.
if [ "$sign" -ne '1' ]
then
if [ "$sign" -ne '1' ]; then
return
fi
@@ -167,34 +157,25 @@ sign() {
sign_os="$1"
sign_bin_path="$2"
if [ "$sign_os" != 'windows' ]
then
gpg\
--default-key "$gpg_key"\
--detach-sig\
--passphrase "$gpg_key_passphrase"\
--pinentry-mode loopback\
-q\
"$sign_bin_path"\
if [ "$sign_os" != 'windows' ]; then
gpg \
--default-key "$gpg_key" \
--detach-sig \
--passphrase "$gpg_key_passphrase" \
--pinentry-mode loopback -q "$sign_bin_path" \
;
return
# TODO(e.burkov): Enable for all releases.
elif [ "$channel" != 'beta' ]
then
return
elif [ "$channel" = 'beta' ] || [ "$channel" = 'release' ]; then
signed_bin_path="${sign_bin_path}.signed"
env INPUT_FILE="$sign_bin_path" \
OUTPUT_FILE="$signed_bin_path" \
SIGNER_API_KEY="$signer_api_key" \
"$deploy_script_path" sign-executable
mv "$signed_bin_path" "$sign_bin_path"
fi
signed_bin_path="${sign_bin_path}.signed"
env\
INPUT_FILE="$sign_bin_path"\
OUTPUT_FILE="$signed_bin_path"\
SIGNER_API_KEY="$signer_api_key"\
"$deploy_script_path" sign-executable\
;
mv "$signed_bin_path" "$sign_bin_path"
}
# Function build builds the release for one platform. It builds a binary and an
@@ -202,17 +183,16 @@ sign() {
build() {
# Get the arguments. Here and below, use the "build_" prefix for all
# variables local to function build.
build_dir="${dist}/${1}/AdGuardHome"\
build_ar="$2"\
build_os="$3"\
build_arch="$4"\
build_arm="$5"\
build_mips="$6"\
build_dir="${dist}/${1}/AdGuardHome" \
build_ar="$2" \
build_os="$3" \
build_arch="$4" \
build_arm="$5" \
build_mips="$6" \
;
# Use the ".exe" filename extension if we build a Windows release.
if [ "$build_os" = 'windows' ]
then
if [ "$build_os" = 'windows' ]; then
build_output="./${build_dir}/AdGuardHome.exe"
else
build_output="./${build_dir}/AdGuardHome"
@@ -224,16 +204,14 @@ build() {
#
# Set GOARM and GOMIPS to an empty string if $build_arm and $build_mips are
# the zero value by removing the hyphen as if it's a prefix.
env\
GOARCH="$build_arch"\
GOARM="${build_arm#-}"\
GOMIPS="${build_mips#-}"\
GOOS="$os"\
VERBOSE="$(( verbose - 1 ))"\
VERSION="$version"\
OUT="$build_output"\
sh ./scripts/make/go-build.sh\
;
env GOARCH="$build_arch" \
GOARM="${build_arm#-}" \
GOMIPS="${build_mips#-}" \
GOOS="$os" \
VERBOSE="$((verbose - 1))" \
VERSION="$version" \
OUT="$build_output" \
sh ./scripts/make/go-build.sh
log "$build_output"
@@ -244,17 +222,16 @@ build() {
# Make archives. Windows and macOS prefer ZIP archives; the rest,
# gzipped tarballs.
case "$build_os"
in
('darwin'|'windows')
case "$build_os" in
'darwin' | 'windows')
build_archive="./${dist}/${build_ar}.zip"
# TODO(a.garipov): Find an option similar to the -C option of tar for
# zip.
( cd "${dist}/${1}" && zip -9 -q -r "../../${build_archive}" "./AdGuardHome" )
(cd "${dist}/${1}" && zip -9 -q -r "../../${build_archive}" "./AdGuardHome")
;;
(*)
*)
build_archive="./${dist}/${build_ar}.tar.gz"
tar -C "./${dist}/${1}" -c -f - "./AdGuardHome" | gzip -9 - > "$build_archive"
tar -C "./${dist}/${1}" -c -f - "./AdGuardHome" | gzip -9 - >"$build_archive"
;;
esac
@@ -265,8 +242,7 @@ log "starting builds"
# Go over all platforms defined in the space-separated table above, tweak the
# values where necessary, and feed to build.
echo "$platforms" | while read -r os arch arm mips
do
echo "$platforms" | while read -r os arch arm mips; do
# See if the architecture or the OS is in the allowlist. To do so, try
# removing everything that matches the pattern (well, a prefix, but that
# doesn't matter here) containing the arch or the OS.
@@ -277,29 +253,28 @@ do
# "* windows *", which doesn't match, so nothing is removed.
#
# See https://stackoverflow.com/a/43912605/1892060.
if [ "${arches##* $arch *}" != '' ]
then
#
# shellcheck disable=SC2295
if [ "${arches##* $arch *}" != '' ]; then
log "$arch excluded, continuing"
continue
elif [ "${oses##* $os *}" != '' ]
then
elif [ "${oses##* $os *}" != '' ]; then
log "$os excluded, continuing"
continue
fi
case "$arch"
in
(arm)
case "$arch" in
arm)
dir="AdGuardHome_${os}_${arch}_${arm}"
ar="AdGuardHome_${os}_${arch}v${arm}"
;;
(mips*)
mips*)
dir="AdGuardHome_${os}_${arch}_${mips}"
ar="$dir"
;;
(*)
*)
dir="AdGuardHome_${os}_${arch}"
ar="$dir"
;;
@@ -311,7 +286,7 @@ done
log "packing frontend"
build_archive="./${dist}/AdGuardHome_frontend.tar.gz"
tar -c -f - ./build | gzip -9 - > "$build_archive"
tar -c -f - ./build | gzip -9 - >"$build_archive"
log "$build_archive"
log "calculating checksums"
@@ -319,8 +294,7 @@ log "calculating checksums"
# calculate_checksums uses the previously detected SHA-256 tool to calculate
# checksums. Do not use find with -exec, since shasum requires arguments.
calculate_checksums() {
if [ "$use_shasum" -eq '0' ]
then
if [ "$use_shasum" -eq '0' ]; then
sha256sum "$@"
else
shasum -a 256 "$@"
@@ -337,24 +311,22 @@ calculate_checksums() {
cd "./${dist}"
: > ./checksums.txt
: >./checksums.txt
for archive in ./*.zip ./*.tar.gz
do
for archive in ./*.zip ./*.tar.gz; do
# Make sure that we don't try to calculate a checksum for a glob pattern
# that matched no files.
if [ ! -f "$archive" ]
then
if [ ! -f "$archive" ]; then
continue
fi
calculate_checksums "$archive" >> ./checksums.txt
calculate_checksums "$archive" >>./checksums.txt
done
)
log "writing versions"
echo "version=$version" > "./${dist}/version.txt"
echo "version=$version" >"./${dist}/version.txt"
# Create the version.json file.
@@ -364,8 +336,7 @@ readonly version_download_url version_json
# If the channel is edge, point users to the "Platforms" page on the Wiki,
# because the direct links to the edge packages are listed there.
if [ "$channel" = 'edge' ]
then
if [ "$channel" = 'edge' ]; then
announcement_url='https://github.com/AdguardTeam/AdGuardHome/wiki/Platforms'
else
announcement_url="https://github.com/AdguardTeam/AdGuardHome/releases/tag/${version}"
@@ -379,7 +350,7 @@ echo "{
\"announcement\": \"AdGuard Home ${version} is now available!\",
\"announcement_url\": \"${announcement_url}\",
\"selfupdate_min_version\": \"0.0\",
" >> "$version_json"
" >>"$version_json"
# Add the MIPS* object keys without the "softfloat" part to mitigate the
# consequences of #5373.
@@ -389,18 +360,17 @@ echo "
\"download_linux_mips64\": \"${version_download_url}/AdGuardHome_linux_mips64_softfloat.tar.gz\",
\"download_linux_mips64le\": \"${version_download_url}/AdGuardHome_linux_mips64le_softfloat.tar.gz\",
\"download_linux_mipsle\": \"${version_download_url}/AdGuardHome_linux_mipsle_softfloat.tar.gz\",
" >> "$version_json"
" >>"$version_json"
# Same as with checksums above, don't use ls, because files matching one of the
# patterns may be absent.
ar_files="$( find "./${dist}/" ! -name "${dist}" -prune \( -name '*.tar.gz' -o -name '*.zip' \) )"
ar_files_len="$( echo "$ar_files" | wc -l )"
ar_files="$(find "./${dist}" ! -name "${dist}" -prune \( -name '*.tar.gz' -o -name '*.zip' \))"
ar_files_len="$(echo "$ar_files" | wc -l)"
readonly ar_files ar_files_len
i='1'
# Don't use quotes to get word splitting.
for f in $ar_files
do
for f in $ar_files; do
platform="$f"
# Remove the prefix.
@@ -413,16 +383,15 @@ do
# Use the filename's base path.
filename="${f#"./${dist}/"}"
if [ "$i" -eq "$ar_files_len" ]
then
echo " \"download_${platform}\": \"${version_download_url}/${filename}\"" >> "$version_json"
if [ "$i" -eq "$ar_files_len" ]; then
echo " \"download_${platform}\": \"${version_download_url}/${filename}\"" >>"$version_json"
else
echo " \"download_${platform}\": \"${version_download_url}/${filename}\"," >> "$version_json"
echo " \"download_${platform}\": \"${version_download_url}/${filename}\"," >>"$version_json"
fi
i="$(( i + 1 ))"
i="$((i + 1))"
done
echo '}' >> "$version_json"
echo '}' >>"$version_json"
log "finished"

View File

@@ -1,29 +0,0 @@
#!/bin/sh
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '0' ]
then
set -x
fi
set -e -f -u
dist_dir="${DIST_DIR:?please set DIST_DIR}"
sudo_cmd="${SUDO:-}"
readonly dist_dir sudo_cmd
$sudo_cmd rm -f\
./AdGuardHome\
./AdGuardHome.exe\
./coverage.txt\
;
$sudo_cmd rm -f -r\
./bin/\
./build/static/\
./client/node_modules/\
./data/\
"./${dist_dir}/"\
;

View File

@@ -7,13 +7,11 @@ readonly verbose
# 0 = Don't print anything except for errors.
# 1 = Print commands, but not nested commands.
# 2 = Print everything.
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
set -x
v_flags='-v=1'
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
elif [ "$verbose" -gt '0' ]; then
set -x
v_flags='-v=1'
x_flags='-x=0'
@@ -26,8 +24,7 @@ readonly v_flags x_flags
set -e -f -u
if [ "${RACE:-1}" -eq '0' ]
then
if [ "${RACE:-1}" -eq '0' ]; then
race_flags='--race=0'
else
race_flags='--race=1'
@@ -36,20 +33,20 @@ readonly race_flags
go="${GO:-go}"
count_flags='--count=1'
count_flags='--count=2'
shuffle_flags='--shuffle=on'
timeout_flags="${TIMEOUT_FLAGS:---timeout=30s}"
readonly go count_flags shuffle_flags timeout_flags
"$go" test\
"$count_flags"\
"$shuffle_flags"\
"$race_flags"\
"$timeout_flags"\
"$x_flags"\
"$v_flags"\
--bench='.'\
--benchmem\
--benchtime=1s\
--run='^$'\
"$go" test \
"$count_flags" \
"$shuffle_flags" \
"$race_flags" \
"$timeout_flags" \
"$x_flags" \
"$v_flags" \
--bench='.' \
--benchmem \
--benchtime='1s' \
--run='^$' \
./...

View File

@@ -9,7 +9,7 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 1
# AdGuard-Project-Version: 2
# The default verbosity level is 0. Show every command that is run and every
# package that is processed if the caller requested verbosity level greater than
@@ -18,14 +18,12 @@
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
env
set -x
v_flags='-v=1'
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
elif [ "$verbose" -gt '0' ]; then
set -x
v_flags='-v=1'
x_flags='-x=0'
@@ -49,13 +47,12 @@ readonly go
channel="${CHANNEL:?please set CHANNEL}"
readonly channel
case "$channel"
in
('development'|'edge'|'beta'|'release'|'candidate')
case "$channel" in
'development' | 'edge' | 'beta' | 'release' | 'candidate')
# All is well, go on.
;;
(*)
echo "invalid channel '$channel', supported values are\
*)
echo "invalid channel '$channel', supported values are \
'development', 'edge', 'beta', 'release', and 'candidate'" 1>&2
exit 1
;;
@@ -64,14 +61,13 @@ esac
# Check VERSION against the default value from the Makefile. If it is that, use
# the version calculation script.
version="${VERSION:-}"
if [ "$version" = 'v0.0.0' ] || [ "$version" = '' ]
then
version="$( sh ./scripts/make/version.sh )"
if [ "$version" = 'v0.0.0' ] || [ "$version" = '' ]; then
version="$(sh ./scripts/make/version.sh)"
fi
readonly version
# Set date and time of the latest commit unless already set.
committime="${SOURCE_DATE_EPOCH:-$( git log -1 --pretty=%ct )}"
committime="${SOURCE_DATE_EPOCH:-$(git log -1 --pretty=%ct)}"
readonly committime
# Set the linker flags accordingly: set the release channel and the current
@@ -84,11 +80,9 @@ ldflags="-s -w"
ldflags="${ldflags} -X ${version_pkg}.version=${version}"
ldflags="${ldflags} -X ${version_pkg}.channel=${channel}"
ldflags="${ldflags} -X ${version_pkg}.committime=${committime}"
if [ "${GOARM:-}" != '' ]
then
if [ "${GOARM:-}" != '' ]; then
ldflags="${ldflags} -X ${version_pkg}.goarm=${GOARM}"
elif [ "${GOMIPS:-}" != '' ]
then
elif [ "${GOMIPS:-}" != '' ]; then
ldflags="${ldflags} -X ${version_pkg}.gomips=${GOMIPS}"
fi
readonly ldflags
@@ -99,9 +93,8 @@ readonly parallelism
# Use GOFLAGS for -p, because -p=0 simply disables the build instead of leaving
# the default value.
if [ "${parallelism}" != '' ]
then
GOFLAGS="${GOFLAGS:-} -p=${parallelism}"
if [ "${parallelism}" != '' ]; then
GOFLAGS="${GOFLAGS:-} -p=${parallelism}"
fi
readonly GOFLAGS
export GOFLAGS
@@ -115,8 +108,7 @@ readonly o_flags
# Allow users to enable the race detector. Unfortunately, that means that cgo
# must be enabled.
if [ "${RACE:-0}" -eq '0' ]
then
if [ "${RACE:-0}" -eq '0' ]; then
CGO_ENABLED='0'
race_flags='--race=0'
else
@@ -130,24 +122,23 @@ GO111MODULE='on'
export GO111MODULE
# Build the new binary if requested.
if [ "${NEXTAPI:-0}" -eq '0' ]
then
if [ "${NEXTAPI:-0}" -eq '0' ]; then
tags_flags='--tags='
else
tags_flags='--tags=next'
fi
readonly tags_flags
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
"$go" env
fi
"$go" build\
--ldflags="$ldflags"\
"$race_flags"\
"$tags_flags"\
--trimpath\
"$o_flags"\
"$v_flags"\
"$x_flags"
"$go" build \
--ldflags="$ldflags" \
"$race_flags" \
"$tags_flags" \
--trimpath \
"$o_flags" \
"$v_flags" \
"$x_flags" \
;

View File

@@ -3,18 +3,16 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 1
# AdGuard-Project-Version: 2
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
env
set -x
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
elif [ "$verbose" -gt '0' ]; then
set -x
x_flags='-x=0'
else

View File

@@ -7,13 +7,11 @@ readonly verbose
# 0 = Don't print anything except for errors.
# 1 = Print commands, but not nested commands.
# 2 = Print everything.
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
set -x
v_flags='-v=1'
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
elif [ "$verbose" -gt '0' ]; then
set -x
v_flags='-v=1'
x_flags='-x=0'
@@ -26,8 +24,7 @@ readonly v_flags x_flags
set -e -f -u
if [ "${RACE:-1}" -eq '0' ]
then
if [ "${RACE:-1}" -eq '0' ]; then
race_flags='--race=0'
else
race_flags='--race=1'
@@ -36,7 +33,7 @@ readonly race_flags
go="${GO:-go}"
count_flags='--count=1'
count_flags='--count=2'
shuffle_flags='--shuffle=on'
timeout_flags="${TIMEOUT_FLAGS:---timeout=30s}"
fuzztime_flags="${FUZZTIME_FLAGS:---fuzztime=20s}"
@@ -44,15 +41,15 @@ fuzztime_flags="${FUZZTIME_FLAGS:---fuzztime=20s}"
readonly go count_flags shuffle_flags timeout_flags fuzztime_flags
# TODO(a.garipov): File an issue about using --fuzz with multiple packages.
"$go" test\
"$count_flags"\
"$shuffle_flags"\
"$race_flags"\
"$timeout_flags"\
"$x_flags"\
"$v_flags"\
"$fuzztime_flags"\
--fuzz='.'\
--run='^$'\
./internal/filtering/rulelist/\
"$go" test \
"$count_flags" \
"$shuffle_flags" \
"$race_flags" \
"$timeout_flags" \
"$x_flags" \
"$v_flags" \
"$fuzztime_flags" \
--fuzz='.' \
--run='^$' \
./internal/filtering/rulelist/ \
;

View File

@@ -3,19 +3,17 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 8
# AdGuard-Project-Version: 13
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
# Set $EXIT_ON_ERROR to zero to see all errors.
if [ "${EXIT_ON_ERROR:-1}" -eq '0' ]
then
if [ "${EXIT_ON_ERROR:-1}" -eq '0' ]; then
set +e
else
set -e
@@ -23,23 +21,26 @@ fi
set -f -u
# Source the common helpers, including not_found and run_linter.
. ./scripts/make/helper.sh
# Simple analyzers
# blocklist_imports is a simple check against unwanted packages. The following
# packages are banned:
#
# * Package errors is replaced by our own package in the
# github.com/AdguardTeam/golibs module.
# github.com/AdguardTeam/golibs module.
#
# * Packages golang.org/x/exp/slices and golang.org/x/net/context have been
# moved into stdlib.
# * Packages log and github.com/AdguardTeam/golibs/log are replaced by
# stdlib's new package log/slog and AdGuard's new utilities package
# github.com/AdguardTeam/golibs/logutil/slogutil.
#
# * Package github.com/prometheus/client_golang/prometheus/promauto is not
# recommended, as it encourages reliance on global state.
#
# * Packages golang.org/x/exp/maps, golang.org/x/exp/slices, and
# golang.org/x/net/context have been moved into stdlib.
#
# * Package io/ioutil is soft-deprecated.
#
@@ -54,48 +55,60 @@ set -f -u
#
# * Package unsafe is… unsafe.
#
# If your project needs more exceptions, add and document them. Currently,
# there are only two standard exceptions:
# Currently, the only standard exception are files generated from protobuf
# schemas, which use package reflect. If your project needs more exceptions,
# add and document them.
#
# * Files generated from protobuf schemas, which use package reflect.
# NOTE: Flag -H for grep is non-POSIX but all of Busybox, GNU, macOS, and
# OpenBSD support it.
#
# * Windows-specific code caused by golang.org/x/sys/windows API design.
# NOTE: Exclude the security_windows.go, because it requires unsafe for the OS
# APIs.
#
# TODO(a.garipov): Add golibs/log.
#
# TODO(a.garipov): Add deprecated package golang.org/x/exp/maps once all
# projects switch to Go 1.23.
blocklist_imports() {
git grep\
-e '[[:space:]]"errors"$'\
-e '[[:space:]]"golang.org/x/exp/slices"$'\
-e '[[:space:]]"golang.org/x/net/context"$'\
-e '[[:space:]]"io/ioutil"$'\
-e '[[:space:]]"log"$'\
-e '[[:space:]]"reflect"$'\
-e '[[:space:]]"sort"$'\
-e '[[:space:]]"unsafe"$'\
-n\
-- '*.go'\
':!*.pb.go'\
':!./internal/aghos/permission_windows.go'\
| sed -e 's/^\([^[:space:]]\+\)\(.*\)$/\1 blocked import:\2/'\
|| exit 0
find . \
-type 'f' \
-name '*.go' \
'!' '(' \
-name '*.pb.go' \
-o -path './internal/permcheck/security_windows.go' \
')' \
-exec \
'grep' \
'-H' \
'-e' '[[:space:]]"errors"$' \
'-e' '[[:space:]]"github.com/prometheus/client_golang/prometheus/promauto"$' \
'-e' '[[:space:]]"golang.org/x/exp/maps"$' \
'-e' '[[:space:]]"golang.org/x/exp/slices"$' \
'-e' '[[:space:]]"golang.org/x/net/context"$' \
'-e' '[[:space:]]"io/ioutil"$' \
'-e' '[[:space:]]"log"$' \
'-e' '[[:space:]]"reflect"$' \
'-e' '[[:space:]]"sort"$' \
'-e' '[[:space:]]"unsafe"$' \
'-n' \
'{}' \
';'
}
# method_const is a simple check against the usage of some raw strings and
# numbers where one should use named constants.
method_const() {
git grep -F\
-e '"DELETE"'\
-e '"GET"'\
-e '"PATCH"'\
-e '"POST"'\
-e '"PUT"'\
-n\
-- '*.go'\
| sed -e 's/^\([^[:space:]]\+\)\(.*\)$/\1 http method literal:\2/'\
|| exit 0
find . \
-type 'f' \
-name '*.go' \
-exec \
'grep' \
'-H' \
'-e' '"DELETE"' \
'-e' '"GET"' \
'-e' '"PATCH"' \
'-e' '"POST"' \
'-e' '"PUT"' \
'-n' \
'{}' \
';'
}
# underscores is a simple check against Go filenames with underscores. Add new
@@ -103,35 +116,34 @@ method_const() {
# use of filenames like client_manager.go.
underscores() {
underscore_files="$(
git ls-files '*_*.go'\
| grep -F\
-e '_bsd.go'\
-e '_darwin.go'\
-e '_freebsd.go'\
-e '_generate.go'\
-e '_linux.go'\
-e '_next.go'\
-e '_openbsd.go'\
-e '_others.go'\
-e '_test.go'\
-e '_unix.go'\
-e '_windows.go'\
-v\
| sed -e 's/./\t\0/'
find . \
-type 'f' \
-name '*_*.go' \
'!' '(' -name '*_bsd.go' \
-o -name '*_darwin.go' \
-o -name '*_freebsd.go' \
-o -name '*_generate.go' \
-o -name '*_linux.go' \
-o -name '*_next.go' \
-o -name '*_openbsd.go' \
-o -name '*_others.go' \
-o -name '*_test.go' \
-o -name '*_unix.go' \
-o -name '*_windows.go' \
')' \
-exec 'printf' '\t%s\n' '{}' ';'
)"
readonly underscore_files
if [ "$underscore_files" != '' ]
then
echo 'found file names with underscores:'
echo "$underscore_files"
if [ "$underscore_files" != '' ]; then
printf \
'found file names with underscores:\n%s\n' \
"$underscore_files"
fi
}
# TODO(a.garipov): Add an analyzer to look for `fallthrough`, `goto`, and `new`?
# Checks
run_linter -e blocklist_imports
@@ -142,8 +154,6 @@ run_linter -e underscores
run_linter -e gofumpt --extra -e -l .
# TODO(a.garipov): golint is deprecated, find a suitable replacement.
run_linter "${GO:-go}" vet ./...
run_linter govulncheck ./...
@@ -151,129 +161,138 @@ run_linter govulncheck ./...
run_linter gocyclo --over 10 .
# TODO(a.garipov): Enable 10 for all.
run_linter gocognit --over='20'\
./internal/querylog/\
run_linter gocognit --over='20' \
./internal/querylog/ \
;
run_linter gocognit --over='19'\
./internal/home/\
run_linter gocognit --over='19' \
./internal/home/ \
;
run_linter gocognit --over='18'\
./internal/aghtls/\
run_linter gocognit --over='18' \
./internal/aghtls/ \
;
run_linter gocognit --over='15'\
./internal/aghos/\
./internal/filtering/\
run_linter gocognit --over='15' \
./internal/aghos/ \
./internal/filtering/ \
;
run_linter gocognit --over='14'\
./internal/dhcpd\
run_linter gocognit --over='14' \
./internal/dhcpd \
;
run_linter gocognit --over='13'\
./internal/aghnet/\
run_linter gocognit --over='13' \
./internal/aghnet/ \
;
run_linter gocognit --over='12'\
./internal/filtering/rewrite/\
run_linter gocognit --over='12' \
./internal/filtering/rewrite/ \
;
run_linter gocognit --over='11'\
./internal/updater/\
run_linter gocognit --over='11' \
./internal/updater/ \
;
run_linter gocognit --over='10'\
./internal/aghalg/\
./internal/aghhttp/\
./internal/aghrenameio/\
./internal/aghtest/\
./internal/arpdb/\
./internal/client/\
./internal/configmigrate/\
./internal/dhcpsvc\
./internal/dnsforward/\
./internal/filtering/hashprefix/\
./internal/filtering/rulelist/\
./internal/filtering/safesearch/\
./internal/ipset\
./internal/next/\
./internal/rdns/\
./internal/schedule/\
./internal/stats/\
./internal/tools/\
./internal/version/\
./internal/whois/\
./scripts/\
run_linter gocognit --over='10' \
./internal/aghalg/ \
./internal/aghhttp/ \
./internal/aghrenameio/ \
./internal/aghtest/ \
./internal/arpdb/ \
./internal/client/ \
./internal/configmigrate/ \
./internal/dhcpsvc \
./internal/dnsforward/ \
./internal/filtering/hashprefix/ \
./internal/filtering/rulelist/ \
./internal/filtering/safesearch/ \
./internal/ipset \
./internal/next/ \
./internal/rdns/ \
./internal/schedule/ \
./internal/stats/ \
./internal/tools/ \
./internal/version/ \
./internal/whois/ \
./scripts/ \
;
run_linter ineffassign ./...
run_linter unparam ./...
git ls-files -- 'Makefile' '*.conf' '*.go' '*.mod' '*.sh' '*.yaml' '*.yml'\
| xargs misspell --error\
| sed -e 's/^/misspell: /'
find . \
-type 'f' \
'(' \
-name 'Makefile' \
-o -name '*.conf' \
-o -name '*.go' \
-o -name '*.mod' \
-o -name '*.sh' \
-o -name '*.yaml' \
-o -name '*.yml' \
')' \
-exec 'misspell' '--error' '{}' '+'
run_linter nilness ./...
# TODO(a.garipov): Enable for all.
run_linter fieldalignment \
./internal/aghalg/\
./internal/aghhttp/\
./internal/aghos/\
./internal/aghrenameio/\
./internal/aghtest/\
./internal/aghtls/\
./internal/arpdb/\
./internal/client/\
./internal/configmigrate/\
./internal/dhcpsvc/\
./internal/filtering/hashprefix/\
./internal/filtering/rewrite/\
./internal/filtering/rulelist/\
./internal/filtering/safesearch/\
./internal/ipset/\
./internal/next/...\
./internal/querylog/\
./internal/rdns/\
./internal/schedule/\
./internal/stats/\
./internal/updater/\
./internal/version/\
./internal/whois/\
./internal/aghalg/ \
./internal/aghhttp/ \
./internal/aghos/ \
./internal/aghrenameio/ \
./internal/aghtest/ \
./internal/aghtls/ \
./internal/arpdb/ \
./internal/client/ \
./internal/configmigrate/ \
./internal/dhcpsvc/ \
./internal/filtering/hashprefix/ \
./internal/filtering/rewrite/ \
./internal/filtering/rulelist/ \
./internal/filtering/safesearch/ \
./internal/ipset/ \
./internal/next/... \
./internal/querylog/ \
./internal/rdns/ \
./internal/schedule/ \
./internal/stats/ \
./internal/updater/ \
./internal/version/ \
./internal/whois/ \
;
run_linter -e shadow --strict ./...
# TODO(a.garipov): Enable for all.
# TODO(e.burkov): Re-enable G115.
run_linter gosec --exclude G115 --quiet\
./internal/aghalg/\
./internal/aghhttp/\
./internal/aghnet/\
./internal/aghos/\
./internal/aghrenameio/\
./internal/aghtest/\
./internal/arpdb/\
./internal/client/\
./internal/configmigrate/\
./internal/dhcpd/\
./internal/dhcpsvc/\
./internal/dnsforward/\
./internal/filtering/hashprefix/\
./internal/filtering/rewrite/\
./internal/filtering/rulelist/\
./internal/filtering/safesearch/\
./internal/ipset/\
./internal/next/\
./internal/rdns/\
./internal/schedule/\
./internal/stats/\
./internal/tools/\
./internal/version/\
./internal/whois/\
run_linter gosec --exclude G115 --quiet \
./internal/aghalg/ \
./internal/aghhttp/ \
./internal/aghnet/ \
./internal/aghos/ \
./internal/aghrenameio/ \
./internal/aghtest/ \
./internal/arpdb/ \
./internal/client/ \
./internal/configmigrate/ \
./internal/dhcpd/ \
./internal/dhcpsvc/ \
./internal/dnsforward/ \
./internal/filtering/hashprefix/ \
./internal/filtering/rewrite/ \
./internal/filtering/rulelist/ \
./internal/filtering/safesearch/ \
./internal/ipset/ \
./internal/next/ \
./internal/rdns/ \
./internal/schedule/ \
./internal/stats/ \
./internal/tools/ \
./internal/version/ \
./internal/whois/ \
;
run_linter errcheck ./...
@@ -287,4 +306,4 @@ windows: GOOS=windows
'
readonly staticcheck_matrix
echo "$staticcheck_matrix" | run_linter staticcheck --matrix ./...
printf '%s' "$staticcheck_matrix" | run_linter staticcheck --matrix ./...

View File

@@ -3,7 +3,7 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 1
# AdGuard-Project-Version: 6
verbose="${VERBOSE:-0}"
readonly verbose
@@ -12,13 +12,11 @@ readonly verbose
# 0 = Don't print anything except for errors.
# 1 = Print commands, but not nested commands.
# 2 = Print everything.
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
set -x
v_flags='-v=1'
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
elif [ "$verbose" -gt '0' ]; then
set -x
v_flags='-v=1'
x_flags='-x=0'
@@ -31,29 +29,54 @@ readonly v_flags x_flags
set -e -f -u
if [ "${RACE:-1}" -eq '0' ]
then
if [ "${RACE:-1}" -eq '0' ]; then
race_flags='--race=0'
else
race_flags='--race=1'
fi
readonly race_flags
count_flags='--count=2'
cover_flags='--coverprofile=./cover.out'
go="${GO:-go}"
readonly go
count_flags='--count=1'
cover_flags='--coverprofile=./coverage.txt'
shuffle_flags='--shuffle=on'
timeout_flags="${TIMEOUT_FLAGS:---timeout=90s}"
readonly count_flags cover_flags shuffle_flags timeout_flags
readonly count_flags cover_flags go shuffle_flags timeout_flags
"$go" test\
"$count_flags"\
"$cover_flags"\
"$race_flags"\
"$shuffle_flags"\
"$timeout_flags"\
"$v_flags"\
"$x_flags"\
./...
go_test() {
"$go" test \
"$count_flags" \
"$cover_flags" \
"$race_flags" \
"$shuffle_flags" \
"$timeout_flags" \
"$v_flags" \
"$x_flags" \
./...
}
test_reports_dir="${TEST_REPORTS_DIR:-}"
readonly test_reports_dir
if [ "$test_reports_dir" = '' ]; then
go_test
exit "$?"
fi
mkdir -p "$test_reports_dir"
# NOTE: The pipe ignoring the exit code here is intentional, as go-junit-report
# will set the exit code to be saved.
go_test 2>&1 \
| tee "${test_reports_dir}/test-output.txt"
# Don't fail on errors in exporting, because TEST_REPORTS_DIR is generally only
# not empty in CI, and so the exit code must be preserved to exit with it later.
set +e
go-junit-report \
--in "${test_reports_dir}/test-output.txt" \
--set-exit-code \
>"${test_reports_dir}/test-report.xml"
printf '%s\n' "$?" \
>"${test_reports_dir}/test-exit-code.txt"

View File

@@ -3,18 +3,16 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 4
# AdGuard-Project-Version: 6
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
set -x
v_flags='-v=1'
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
elif [ "$verbose" -gt '0' ]; then
set -x
v_flags='-v=1'
x_flags='-x=0'
@@ -33,45 +31,49 @@ readonly go
# Remove only the actual binaries in the bin/ directory, as developers may add
# their own scripts there. Most commonly, a script named “go” for tools that
# call the go binary and need a particular version.
rm -f\
bin/errcheck\
bin/fieldalignment\
bin/gocognit\
bin/gocyclo\
bin/gofumpt\
bin/gosec\
bin/govulncheck\
bin/ineffassign\
bin/misspell\
bin/nilness\
bin/shadow\
bin/staticcheck\
bin/unparam\
rm -f \
bin/errcheck \
bin/fieldalignment \
bin/go-junit-report \
bin/gocognit \
bin/gocyclo \
bin/gofumpt \
bin/gosec \
bin/govulncheck \
bin/ineffassign \
bin/misspell \
bin/nilness \
bin/shadow \
bin/shfmt \
bin/staticcheck \
bin/unparam \
;
# Reset GOARCH and GOOS to make sure we install the tools for the native
# architecture even when we're cross-compiling the main binary, and also to
# prevent the "cannot install cross-compiled binaries when GOBIN is set" error.
env\
GOARCH=""\
GOBIN="${PWD}/bin"\
GOOS=""\
GOWORK='off'\
"$go" install\
--modfile=./internal/tools/go.mod\
"$v_flags"\
"$x_flags"\
github.com/fzipp/gocyclo/cmd/gocyclo\
github.com/golangci/misspell/cmd/misspell\
github.com/gordonklaus/ineffassign\
github.com/kisielk/errcheck\
github.com/securego/gosec/v2/cmd/gosec\
github.com/uudashr/gocognit/cmd/gocognit\
golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment\
golang.org/x/tools/go/analysis/passes/nilness/cmd/nilness\
golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow\
golang.org/x/vuln/cmd/govulncheck\
honnef.co/go/tools/cmd/staticcheck\
mvdan.cc/gofumpt\
mvdan.cc/unparam\
env \
GOARCH="" \
GOBIN="${PWD}/bin" \
GOOS="" \
GOWORK='off' \
"$go" install \
--modfile=./internal/tools/go.mod \
"$v_flags" \
"$x_flags" \
github.com/fzipp/gocyclo/cmd/gocyclo \
github.com/golangci/misspell/cmd/misspell \
github.com/gordonklaus/ineffassign \
github.com/jstemmer/go-junit-report/v2 \
github.com/kisielk/errcheck \
github.com/securego/gosec/v2/cmd/gosec \
github.com/uudashr/gocognit/cmd/gocognit \
golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment \
golang.org/x/tools/go/analysis/passes/nilness/cmd/nilness \
golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow \
golang.org/x/vuln/cmd/govulncheck \
honnef.co/go/tools/cmd/staticcheck \
mvdan.cc/gofumpt \
mvdan.cc/sh/v3/cmd/shfmt \
mvdan.cc/unparam \
;

View File

@@ -3,18 +3,16 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a significant change is made to this script.
#
# AdGuard-Project-Version: 2
# AdGuard-Project-Version: 3
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '1' ]
then
if [ "$verbose" -gt '1' ]; then
env
set -x
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
elif [ "$verbose" -gt '0' ]; then
set -x
x_flags='-x=0'
else

View File

@@ -8,9 +8,7 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a remarkable change is made to this script.
#
# AdGuard-Project-Version: 3
# AdGuard-Project-Version: 4
# Deferred helpers
@@ -23,8 +21,7 @@ make sure you have installed the linter binaries using:
readonly not_found_msg
not_found() {
if [ "$?" -eq '127' ]
then
if [ "$?" -eq '127' ]; then
# Code 127 is the exit status a shell uses when a command or a file is
# not found, according to the Bash Hackers wiki.
#
@@ -34,8 +31,6 @@ not_found() {
}
trap not_found EXIT
# Helpers
# run_linter runs the given linter with two additions:
@@ -47,8 +42,7 @@ trap not_found EXIT
run_linter() (
set +e
if [ "${VERBOSE:-0}" -lt '2' ]
then
if [ "${VERBOSE:-0}" -lt '2' ]; then
set +x
fi
@@ -56,8 +50,7 @@ run_linter() (
shift
exit_on_output='0'
if [ "$cmd" = '-e' ]
then
if [ "$cmd" = '-e' ]; then
exit_on_output='1'
cmd="${1:?run_linter: provide a command}"
shift
@@ -65,17 +58,15 @@ run_linter() (
readonly cmd
output="$( "$cmd" "$@" )"
output="$("$cmd" "$@")"
exitcode="$?"
readonly output
if [ "$output" != '' ]
then
if [ "$output" != '' ]; then
echo "$output" | sed -e "s/^/${cmd}: /"
if [ "$exitcode" -eq '0' ] && [ "$exit_on_output" -eq '1' ]
then
if [ "$exitcode" -eq '0' ] && [ "$exit_on_output" -eq '1' ]; then
exitcode='1'
fi
fi

View File

@@ -3,21 +3,18 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a remarkable change is made to this script.
#
# AdGuard-Project-Version: 2
# AdGuard-Project-Version: 3
verbose="${VERBOSE:-0}"
readonly verbose
set -e -f -u
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
# NOTE: Adjust for your project.
# markdownlint\
# ./README.md\
# ;
# TODO(e.burkov): Lint markdown documents within this project.
# markdownlint \
# ./README.md \
# ;

View File

@@ -3,7 +3,7 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a remarkable change is made to this script.
#
# AdGuard-Project-Version: 2
# AdGuard-Project-Version: 3
verbose="${VERBOSE:-0}"
readonly verbose
@@ -11,29 +11,25 @@ readonly verbose
# Don't use -f, because we use globs in this script.
set -e -u
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
# NOTE: Adjust for your project.
#
# TODO(e.burkov): Add build-docker.sh, build-release.sh and install.sh.
shellcheck -e 'SC2250' -f 'gcc' -o 'all' -x --\
./scripts/hooks/*\
./scripts/snap/*\
./scripts/make/clean.sh\
./scripts/make/go-bench.sh\
./scripts/make/go-build.sh\
./scripts/make/go-deps.sh\
./scripts/make/go-fuzz.sh\
./scripts/make/go-lint.sh\
./scripts/make/go-test.sh\
./scripts/make/go-tools.sh\
./scripts/make/go-upd-tools.sh\
./scripts/make/helper.sh\
./scripts/make/md-lint.sh\
./scripts/make/sh-lint.sh\
./scripts/make/txt-lint.sh\
./scripts/make/version.sh\
# Source the common helpers, including not_found and run_linter.
. ./scripts/make/helper.sh
run_linter -e shfmt --binary-next-line -d -p -s \
./scripts/hooks/* \
./scripts/install.sh \
./scripts/make/*.sh \
./scripts/snap/*.sh \
./snap/local/*.sh \
;
shellcheck -e 'SC2250' -e 'SC2310' -f 'gcc' -o 'all' -x -- \
./scripts/hooks/* \
./scripts/install.sh \
./scripts/make/*.sh \
./scripts/snap/*.sh \
./snap/local/*.sh \
;

View File

@@ -3,19 +3,17 @@
# This comment is used to simplify checking local copies of the script. Bump
# this number every time a remarkable change is made to this script.
#
# AdGuard-Project-Version: 5
# AdGuard-Project-Version: 8
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
# Set $EXIT_ON_ERROR to zero to see all errors.
if [ "${EXIT_ON_ERROR:-1}" -eq '0' ]
then
if [ "${EXIT_ON_ERROR:-1}" -eq '0' ]; then
set +e
else
set -e
@@ -32,19 +30,30 @@ set -f -u
# trailing_newlines is a simple check that makes sure that all plain-text files
# have a trailing newlines to make sure that all tools work correctly with them.
trailing_newlines() (
nl="$( printf "\n" )"
nl="$(printf '\n')"
readonly nl
# NOTE: Adjust for your project.
git ls-files\
':!*.png'\
':!*.tar.gz'\
':!*.zip'\
| while read -r f
do
final_byte="$( tail -c -1 "$f" )"
if [ "$final_byte" != "$nl" ]
then
find . \
-type 'f' \
'!' '(' \
-name '*.db' \
-o -name '*.exe' \
-o -name '*.out' \
-o -name '*.png' \
-o -name '*.svg' \
-o -name '*.tar.gz' \
-o -name '*.test' \
-o -name '*.zip' \
-o -name 'AdGuardHome' \
-o -name 'adguard-home' \
-o -path '*/node_modules/*' \
-o -path './.git/*' \
-o -path './bin/*' \
-o -path './build/*' \
')' \
| while read -r f; do
final_byte="$(tail -c -1 "$f")"
if [ "$final_byte" != "$nl" ]; then
printf '%s: must have a trailing newline\n' "$f"
fi
done
@@ -53,19 +62,26 @@ trailing_newlines() (
# trailing_whitespace is a simple check that makes sure that there are no
# trailing whitespace in plain-text files.
trailing_whitespace() {
# NOTE: Adjust for your project.
git ls-files\
':!*.bmp'\
':!*.jpg'\
':!*.mmdb'\
':!*.png'\
':!*.svg'\
':!*.tar.gz'\
':!*.webp'\
':!*.zip'\
| while read -r f
do
grep -e '[[:space:]]$' -n -- "$f"\
find . \
-type 'f' \
'!' '(' \
-name '*.db' \
-o -name '*.exe' \
-o -name '*.out' \
-o -name '*.png' \
-o -name '*.svg' \
-o -name '*.tar.gz' \
-o -name '*.test' \
-o -name '*.zip' \
-o -name 'AdGuardHome' \
-o -name 'adguard-home' \
-o -path '*/node_modules/*' \
-o -path './.git/*' \
-o -path './bin/*' \
-o -path './build/*' \
')' \
| while read -r f; do
grep -e '[[:space:]]$' -n -- "$f" \
| sed -e "s:^:${f}\::" -e 's/ \+$/>>>&<<</'
done
}
@@ -74,7 +90,18 @@ run_linter -e trailing_newlines
run_linter -e trailing_whitespace
git ls-files -- '*.conf' '*.md' '*.txt' '*.yaml' '*.yml'\
'client/src/__locales/en.json'\
| xargs misspell --error\
| sed -e 's/^/misspell: /'
find . \
-type 'f' \
'!' '(' \
-path '*/node_modules/*' \
-o -path './data/filters/*' \
')' \
'(' \
-name 'Makefile' \
-o -name '*.conf' \
-o -name '*.md' \
-o -name '*.txt' \
-o -name '*.yaml' \
-o -name '*.yml' \
')' \
-exec 'misspell' '--error' '{}' '+'

View File

@@ -25,8 +25,7 @@
verbose="${VERBOSE:-0}"
readonly verbose
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
@@ -62,20 +61,20 @@ get_last_minor_zero() {
# second field (",2"). The sort is also numeric and reverse ("nr").
#
# Finally, get the top (that is, most recent) version.
git tag\
| grep -e 'v[0-9]\+\.[0-9]\+\.0$'\
| sort -k 1.2,1nr -k 2,2nr -t '.'\
| head -n 1
git tag \
| grep -e 'v[0-9]\+\.[0-9]\+\.0$' \
| sort -k 1.2,1nr -k 2,2nr -t '.' \
| head -n 1 \
;
}
channel="${CHANNEL:?please set CHANNEL}"
readonly channel
case "$channel"
in
('development')
case "$channel" in
'development')
# commit_number is the number of current commit within the branch.
commit_number="$( git rev-list --count master..HEAD )"
commit_number="$(git rev-list --count master..HEAD)"
readonly commit_number
# The development builds are described with a combination of unset semantic
@@ -83,21 +82,21 @@ in
#
# v0.0.0-dev.5-a1b2c3d4
#
version="v0.0.0-dev.${commit_number}+$( git rev-parse --short HEAD )"
version="v0.0.0-dev.${commit_number}+$(git rev-parse --short HEAD)"
;;
('edge')
'edge')
# last_minor_zero is the last new minor release.
last_minor_zero="$( get_last_minor_zero )"
last_minor_zero="$(get_last_minor_zero)"
readonly last_minor_zero
# num_commits_since_minor is the number of commits since the last new
# minor release. If the current commit is the new minor release,
# num_commits_since_minor is zero.
num_commits_since_minor="$( git rev-list --count "${last_minor_zero}..HEAD" )"
num_commits_since_minor="$(git rev-list --count "${last_minor_zero}..HEAD")"
readonly num_commits_since_minor
# next_minor is the next minor release version.
next_minor="$( echo "$last_minor_zero" | awk -F '.' "$bump_minor" )"
next_minor="$(echo "$last_minor_zero" | awk -F '.' "$bump_minor")"
readonly next_minor
# Make this commit a prerelease version for the next minor release. For
@@ -106,21 +105,20 @@ in
#
# v0.124.0-a.5+a1b2c3d4
#
version="${next_minor}-a.${num_commits_since_minor}+$( git rev-parse --short HEAD )"
version="${next_minor}-a.${num_commits_since_minor}+$(git rev-parse --short HEAD)"
;;
('beta'|'release')
'beta' | 'release')
# current_desc is the description of the current git commit. If the
# current commit is tagged, git describe will show the tag.
current_desc="$( git describe )"
current_desc="$(git describe)"
readonly current_desc
# last_tag is the most recent git tag.
last_tag="$( git describe --abbrev=0 )"
last_tag="$(git describe --abbrev=0)"
readonly last_tag
# Require an actual tag for the beta and final releases.
if [ "$current_desc" != "$last_tag" ]
then
if [ "$current_desc" != "$last_tag" ]; then
echo 'need a tag' 1>&2
exit 1
@@ -128,41 +126,39 @@ in
version="$last_tag"
;;
('candidate')
'candidate')
# This pseudo-channel is used to set a proper versions into release
# candidate builds.
# last_tag is expected to be the latest release tag.
last_tag="$( git describe --abbrev=0 )"
last_tag="$(git describe --abbrev=0)"
readonly last_tag
# current_branch is the name of the branch currently checked out.
current_branch="$( git rev-parse --abbrev-ref HEAD )"
current_branch="$(git rev-parse --abbrev-ref HEAD)"
readonly current_branch
# The branch should be named like:
#
# rc-v12.34.56
#
if ! echo "$current_branch" | grep -E -e '^rc-v[0-9]+\.[0-9]+\.[0-9]+$' -q
then
if ! echo "$current_branch" | grep -E -e '^rc-v[0-9]+\.[0-9]+\.[0-9]+$' -q; then
echo "invalid release candidate branch name '$current_branch'" 1>&2
exit 1
fi
version="${current_branch#rc-}-rc.$( git rev-list --count "$last_tag"..HEAD )"
version="${current_branch#rc-}-rc.$(git rev-list --count "$last_tag"..HEAD)"
;;
(*)
echo "invalid channel '$channel', supported values are\
*)
echo "invalid channel '$channel', supported values are \
'development', 'edge', 'beta', 'release' and 'candidate'" 1>&2
exit 1
;;
esac
# Finally, make sure that we don't output invalid versions.
if ! echo "$version" | grep -E -e '^v[0-9]+\.[0-9]+\.[0-9]+(-(a|b|dev|rc)\.[0-9]+)?(\+[[:xdigit:]]+)?$' -q
then
if ! echo "$version" | grep -E -e '^v[0-9]+\.[0-9]+\.[0-9]+(-(a|b|dev|rc)\.[0-9]+)?(\+[[:xdigit:]]+)?$' -q; then
echo "generated an invalid version '$version'" 1>&2
exit 1

View File

@@ -2,8 +2,7 @@
verbose="${VERBOSE:-0}"
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
@@ -14,29 +13,27 @@ set -e -f -u
#
# TODO(a.garipov): Add to helpers.sh and use more actively in scripts.
log() {
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
# Don't use quotes to get word splitting.
echo "$1" 1>&2
fi
}
version="$( ./AdGuardHome_amd64 --version | cut -d ' ' -f 4 )"
if [ "$version" = '' ]
then
version="$(./AdGuardHome_amd64 --version | cut -d ' ' -f 4)"
if [ "$version" = '' ]; then
log 'empty version from ./AdGuardHome_amd64'
exit 1
fi
readonly version
log "version '$version'"
for arch in\
'i386'\
'amd64'\
'armhf'\
'arm64'
do
for arch in \
'amd64' \
'arm64' \
'armhf' \
'i386'; do
build_output="./AdGuardHome_${arch}"
snap_output="./AdGuardHome_${arch}.snap"
snap_dir="${snap_output}.dir"
@@ -48,25 +45,22 @@ do
cp -r './snap/gui' "${snap_dir}/meta/"
# Create a snap.yaml file, setting the values.
sed\
-e 's/%VERSION%/'"$version"'/'\
-e 's/%ARCH%/'"$arch"'/'\
./snap/snap.tmpl.yaml\
> "${snap_dir}/meta/snap.yaml"
sed \
-e 's/%VERSION%/'"$version"'/' \
-e 's/%ARCH%/'"$arch"'/' \
./snap/snap.tmpl.yaml \
>"${snap_dir}/meta/snap.yaml"
# TODO(a.garipov): The snapcraft tool will *always* write everything,
# including errors, to stdout. And there doesn't seem to be a way to change
# that. So, save the combined output, but only show it when snapcraft
# actually fails.
set +e
snapcraft_output="$(
snapcraft pack "$snap_dir" --output "$snap_output" 2>&1
)"
snapcraft_output="$(snapcraft pack "$snap_dir" --output "$snap_output" 2>&1)"
snapcraft_exit_code="$?"
set -e
if [ "$snapcraft_exit_code" -ne '0' ]
then
if [ "$snapcraft_exit_code" -ne '0' ]; then
log "$snapcraft_output"
exit "$snapcraft_exit_code"
fi

View File

@@ -2,8 +2,7 @@
verbose="${VERBOSE:-0}"
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
@@ -12,18 +11,17 @@ set -e -f -u
channel="${CHANNEL:?please set CHANNEL}"
readonly channel
printf '%s %s\n'\
'386' 'i386'\
'amd64' 'amd64'\
'armv7' 'armhf'\
printf '%s %s\n' \
'386' 'i386' \
'amd64' 'amd64' \
'armv7' 'armhf' \
'arm64' 'arm64' \
| while read -r arch snap_arch
do
release_url="https://static.adtidy.org/adguardhome/${channel}/AdGuardHome_linux_${arch}.tar.gz"
output="./AdGuardHome_linux_${arch}.tar.gz"
| while read -r arch snap_arch; do
release_url="https://static.adtidy.org/adguardhome/${channel}/AdGuardHome_linux_${arch}.tar.gz"
output="./AdGuardHome_linux_${arch}.tar.gz"
curl -o "$output" -v "$release_url"
tar -f "$output" -v -x -z
cp ./AdGuardHome/AdGuardHome "./AdGuardHome_${snap_arch}"
rm -f -r "$output" ./AdGuardHome
done
curl -o "$output" -v "$release_url"
tar -f "$output" -v -x -z
cp ./AdGuardHome/AdGuardHome "./AdGuardHome_${snap_arch}"
rm -f -r "$output" ./AdGuardHome
done

View File

@@ -2,8 +2,7 @@
verbose="${VERBOSE:-0}"
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
set -x
fi
@@ -12,8 +11,7 @@ set -e -f -u
# Function log is an echo wrapper that writes to stderr if the caller requested
# verbosity level greater than 0. Otherwise, it does nothing.
log() {
if [ "$verbose" -gt '0' ]
then
if [ "$verbose" -gt '0' ]; then
# Don't use quotes to get word splitting.
echo "$1" 1>&2
fi
@@ -21,8 +19,7 @@ log() {
# Do not set a new lowercase variable, because the snapcraft tool expects the
# uppercase form.
if [ "${SNAPCRAFT_STORE_CREDENTIALS:-}" = '' ]
then
if [ "${SNAPCRAFT_STORE_CREDENTIALS:-}" = '' ]; then
log 'please set SNAPCRAFT_STORE_CREDENTIALS'
exit 1
@@ -40,12 +37,11 @@ default_timeout='90s'
kill_timeout='120s'
readonly default_timeout kill_timeout
for arch in\
'i386'\
'amd64'\
'armhf'\
'arm64'
do
for arch in \
'amd64' \
'arm64' \
'armhf' \
'i386'; do
snap_file="./AdGuardHome_${arch}.snap"
# Catch the exit code and the combined output to later inspect it.
@@ -53,30 +49,28 @@ do
snapcraft_output="$(
# Use timeout(1) to force snapcraft to quit after a certain time. There
# seems to be no environment variable or flag to force this behavior.
timeout\
--preserve-status\
-k "$kill_timeout"\
-v "$default_timeout"\
"$snapcraft_cmd" upload\
--release="${snapcraft_channel}"\
--quiet\
"${snap_file}"\
timeout \
--preserve-status \
-k "$kill_timeout" \
-v "$default_timeout" \
"$snapcraft_cmd" upload \
--release="${snapcraft_channel}" \
--quiet \
"${snap_file}" \
2>&1
)"
snapcraft_exit_code="$?"
set -e
if [ "$snapcraft_exit_code" -eq '0' ]
then
if [ "$snapcraft_exit_code" -eq '0' ]; then
log "successful upload: ${snapcraft_output}"
continue
fi
# Skip the ones that were failed by a duplicate upload error.
case "$snapcraft_output"
in
(*'A file with this exact same content has already been uploaded'*|\
case "$snapcraft_output" in
*'A file with this exact same content has already been uploaded'* | \
*'Error checking upload uniqueness'*)
log "warning: duplicate upload, skipping"
@@ -84,7 +78,7 @@ do
continue
;;
(*)
*)
echo "unexpected snapcraft upload error: ${snapcraft_output}"
return "$snapcraft_exit_code"

View File

@@ -10,6 +10,7 @@ import (
"encoding/json"
"fmt"
"log/slog"
"maps"
"net/url"
"os"
"os/exec"
@@ -21,7 +22,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/logutil/slogutil"
"golang.org/x/exp/maps"
)
const (
@@ -76,40 +76,30 @@ func main() {
usage("")
}
conf, err := readTwoskyConfig()
check(err)
conf := errors.Must(readTwoskyConfig())
var cli *twoskyClient
switch os.Args[1] {
case "summary":
err = summary(conf.Languages)
errors.Check(summary(conf.Languages))
case "download":
cli, err = conf.toClient()
check(err)
cli = errors.Must(conf.toClient())
err = cli.download(ctx, l)
errors.Check(cli.download(ctx, l))
case "unused":
err = unused(ctx, l, conf.LocalizableFiles[0])
err := unused(ctx, l, conf.LocalizableFiles[0])
errors.Check(err)
case "upload":
cli, err = conf.toClient()
check(err)
cli = errors.Must(conf.toClient())
err = cli.upload()
errors.Check(cli.upload())
case "auto-add":
err = autoAdd(conf.LocalizableFiles[0])
err := autoAdd(conf.LocalizableFiles[0])
errors.Check(err)
default:
usage("unknown command")
}
check(err)
}
// check is a simple error-checking helper for scripts.
func check(err error) {
if err != nil {
panic(err)
}
}
// usage prints usage. If addStr is not empty print addStr and exit with code
@@ -163,15 +153,11 @@ func readTwoskyConfig() (t *twoskyConfig, err error) {
var tsc []twoskyConfig
err = json.Unmarshal(b, &tsc)
if err != nil {
err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
return nil, err
return nil, fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
}
if len(tsc) == 0 {
err = fmt.Errorf("%q is empty", twoskyConfFile)
return nil, err
return nil, fmt.Errorf("%q is empty", twoskyConfFile)
}
conf := tsc[0]
@@ -224,7 +210,8 @@ func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
baseLang = langCode(uLangStr)
}
langs := maps.Keys(t.Languages)
langs := slices.Sorted(maps.Keys(t.Languages))
dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
if dlLangStr == "blocker" {
langs = blockerLangCodes
@@ -295,8 +282,7 @@ func summary(langs languages) (err error) {
size := float64(len(baseLoc))
keys := maps.Keys(langs)
slices.Sort(keys)
keys := slices.Sorted(maps.Keys(langs))
for _, lang := range keys {
name := filepath.Join(localesDir, string(lang)+".json")
@@ -399,10 +385,7 @@ func findUnused(fileNames []string, loc locales) (err error) {
}
}
keys := maps.Keys(loc)
slices.Sort(keys)
for _, v := range keys {
for _, v := range slices.Sorted(maps.Keys(loc)) {
fmt.Println(v)
}

View File

@@ -13,6 +13,7 @@ import (
"os"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/logutil/slogutil"
"github.com/google/renameio/v2/maybe"
)
@@ -22,20 +23,19 @@ func main() {
l := slogutil.New(nil)
urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json"
if v, ok := os.LookupEnv("URL"); ok {
urlStr = v
if s := os.Getenv("URL"); s != "" {
urlStr = s
}
// Validate the URL.
_, err := url.Parse(urlStr)
check(err)
errors.Check(err)
c := &http.Client{
Timeout: 10 * time.Second,
}
resp, err := c.Get(urlStr)
check(err)
resp := errors.Must(c.Get(urlStr))
defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError)
if resp.StatusCode != http.StatusOK {
@@ -44,7 +44,7 @@ func main() {
hlFlt := &hlFilters{}
err = json.NewDecoder(resp.Body).Decode(hlFlt)
check(err)
errors.Check(err)
aghFlt := &aghFilters{
Categories: map[string]*aghFiltersCategory{
@@ -93,11 +93,10 @@ func main() {
enc := json.NewEncoder(buf)
enc.SetIndent("", " ")
err = enc.Encode(aghFlt)
check(err)
errors.Check(enc.Encode(aghFlt))
err = maybe.WriteFile("client/src/helpers/filters/filters.ts", buf.Bytes(), 0o644)
check(err)
errors.Check(err)
}
// jsHeader is the header for the generated JavaScript file. It informs the
@@ -109,13 +108,6 @@ const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go;
export default `
// check is a simple error-checking helper for scripts.
func check(err error) {
if err != nil {
panic(err)
}
}
// hlFilters is the JSON structure for the Hostlists Registry rule list index.
type hlFilters struct {
Filters []*hlFiltersFilter `json:"filters"`