74 Commits

Author SHA1 Message Date
Toby
1de95ed53e Merge pull request #136 from apernet/wip-pcapgo
Add pcap back
2024-05-08 17:40:55 -07:00
Haruue
1934c065ec feat(pcap): impl realtime wait() with time offset 2024-05-08 19:45:10 +08:00
Haruue
301f9af3d4 Revert "ci: install pcap for build 2"
This reverts commit 0daaa32fc6.
2024-05-08 19:26:19 +08:00
Haruue
cb0427bfbb Revert "ci: install pcap for build"
This reverts commit 5e15fd6dd9.
2024-05-08 19:26:15 +08:00
Haruue
7456e5907e refactor(pcap): switch to pcapgo 2024-05-08 19:22:17 +08:00
Haruue
8cab86b924 Reapply "Merge pull request #132 from eddc005/feat-pcap"
This reverts commit 2ac8783eb6.
2024-05-08 19:13:49 +08:00
Toby
3ec5456e86 Merge pull request #135 from apernet/revert-pcap
Revert pcap
2024-05-07 23:46:46 -07:00
Toby
b51ea5fa07 Revert "Merge pull request #134 from apernet/ci-cgo"
This reverts commit 5014523ae0, reversing
changes made to c453020349.
2024-05-07 23:44:00 -07:00
Toby
2ac8783eb6 Revert "Merge pull request #132 from eddc005/feat-pcap"
This reverts commit c453020349, reversing
changes made to 5723490a6c.
2024-05-07 23:43:54 -07:00
Toby
5014523ae0 Merge pull request #134 from apernet/ci-cgo
ci: enable cgo
2024-05-07 23:40:17 -07:00
Toby
dabcc9566c ci: enable cgo 2024-05-07 23:38:28 -07:00
Toby
c453020349 Merge pull request #132 from eddc005/feat-pcap
feat: add support for pcap replay
2024-05-07 23:18:03 -07:00
Toby
0daaa32fc6 ci: install pcap for build 2 2024-05-07 23:13:58 -07:00
Toby
5e15fd6dd9 ci: install pcap for build 2024-05-07 23:12:24 -07:00
Toby
76c0f47832 chore: do not default replay.realtime to true 2024-05-07 23:05:06 -07:00
Toby
70fee14103 chore: format 2024-05-07 22:50:58 -07:00
eddc005
abd7725fed close pcap properly and implement ProtectedDialContext 2024-05-07 21:50:06 +01:00
eddc005
f01b79e625 rebase and remove replayDelay 2024-05-06 23:04:54 +01:00
eddc005
94387450cf feat: add support for pcap replay 2024-05-06 22:53:11 +01:00
Toby
5723490a6c Merge pull request #133 from apernet/fix-timestamp
fix: provide correct timestamp for TCP reassembler
2024-05-06 14:38:35 -07:00
Toby
d7506264ad fix: provide correct timestamp for TCP reassembler 2024-05-06 14:35:31 -07:00
Toby
245ac46b65 Merge pull request #130 from apernet/fix-geo-leak
fix: do not reload geoip/geosite when reloading ruleset to prevent leaking references to streams
2024-04-10 22:57:18 -07:00
Toby
107e29ee20 fix: do not reload geoip/geosite when reloading ruleset to prevent leaking references to streams 2024-04-10 21:30:37 -07:00
Toby
5f447d4e31 Merge pull request #124 from apernet/wip-tcp-flush
feat: TCP timeout flush
2024-04-09 11:07:20 -07:00
Toby
347667a2bd feat: TCP timeout flush 2024-04-08 11:54:35 -07:00
Toby
393c29bd2d Merge pull request #123 from apernet/wip-lookup
feat: dns lookup function
2024-04-07 17:49:33 -07:00
Toby
9c0893c512 feat: added protected dial support, removed multi-IO support for simplicity 2024-04-06 14:42:45 -07:00
Toby
ae34b4856a feat: dns lookup function 2024-04-03 20:02:57 -07:00
Toby
d7737e9211 Merge pull request #119 from apernet/update-readme
docs: update readme feature list
2024-04-01 21:50:07 -07:00
Toby
dd9ecc3dd7 docs: update readme feature list 2024-04-01 21:49:06 -07:00
Toby
980d59ed2e Merge pull request #118 from apernet/ci-release
ci: rework release workflow
2024-04-01 20:55:22 -07:00
Toby
af14adf313 ci: rework release workflow 2024-04-01 20:54:24 -07:00
Toby
ab28fc238d Merge pull request #114 from KujouRinka/feat-openvpn
feat: add openVPN analyzer
2024-04-01 20:47:30 -07:00
Toby
e535769086 fix: make sure rx_pkt_cnt & tx_pkt_cnt both always exist 2024-03-30 14:30:20 -07:00
Toby
ecd60d0ff1 chore: improve case spelling 2024-03-30 13:29:19 -07:00
Toby
98264d9e27 chore: format 2024-03-30 13:23:46 -07:00
Toby
bb5d4e32ff Merge pull request #116 from apernet/update-trojan
feat: further improve trojan detection
2024-03-29 23:19:36 -07:00
Toby
ca574393d3 feat: further improve trojan detection 2024-03-29 23:02:20 -07:00
KujouRinka
0e2ee36865 refactor: merge openvpn_tcp and openvpn_udp to openvpn 2024-03-30 12:00:17 +08:00
Toby
b02738cde8 Merge pull request #115 from apernet/update-readme
docs: move to website
2024-03-29 13:08:14 -07:00
Toby
0735fa831d docs: move to website 2024-03-29 13:06:29 -07:00
KujouRinka
2232b553b3 chore: fix typo 2024-03-29 21:37:09 +08:00
KujouRinka
b2f6dec909 feat: add openVPN analyzer 2024-03-29 20:29:30 +08:00
Toby
47a3c9875c Merge pull request #112 from apernet/docs-dep
docs: update dependency install command for openwrt
2024-03-24 11:42:10 -07:00
Toby
4e604904af docs: update dependency install command for openwrt 2024-03-24 11:41:17 -07:00
Toby
bf2988116a Merge pull request #109 from apernet/wip-io-rst
feat: io tcp reset support (forward only)
2024-03-21 18:43:43 -07:00
Haruue
ef1416274d refactor(io): nft/ipt generator func 2024-03-22 02:04:25 +08:00
Toby
57c818038c feat: io tcp reset support (forward only) 2024-03-20 19:01:26 -07:00
Toby
6ad7714c9a Merge pull request #108 from apernet/fix-trojan-doc
docs: remove trojan-killer ref
2024-03-20 18:41:45 -07:00
Toby
ff9c4ccf79 docs: remove trojan-killer ref 2024-03-20 18:40:49 -07:00
Toby
e1d9406fdb Merge pull request #107 from apernet/fix-trojan
fix: trojan analyzer heuristics
2024-03-20 18:39:05 -07:00
Toby
b8e5079b8a fix: trojan analyzer heuristics 2024-03-20 18:37:43 -07:00
Toby
f3b72895ad Merge pull request #106 from apernet/wip-new-trojan
feat: new heuristics for trojan analyzer
2024-03-20 18:11:09 -07:00
Toby
0732dfa7a5 docs: no longer use trojan-killer 2024-03-20 18:10:22 -07:00
Toby
9d96acd8db feat: new heuristics for trojan analyzer 2024-03-20 18:07:26 -07:00
Toby
d1775184ce Merge pull request #102 from macie/badge_fix
docs: Fix status badge URL
2024-03-14 00:37:37 -07:00
macie
05d56616fc ci: Add readable name for workflow
It is visible on Workflows list inside GH Actions tab.
2024-03-14 08:14:30 +01:00
macie
ede70e1a87 docs: Fix badge URL 2024-03-14 08:06:27 +01:00
Toby
920783bd65 Merge pull request #98 from macie/ci
CI/CD: New workflow for testing and static analysis
2024-03-13 19:29:36 -07:00
macie
3a45461c19 test: Remove broken integration test
This tests fails when there are no specific binary files, but there is
no method to get the files during test.

Marking the tests as a skipped will gives us a false sense of security. So it's
better to remove it at all.
2024-03-12 12:15:10 +01:00
macie
3022bde81b fix: Linter errors
Fixed:
- "ineffective break statement. Did you mean to break out of the outer loop?" (SA4011)
- "channels used with os/signal.Notify should be buffered" (SA1017)
- "os.Kill cannot be trapped (did you mean syscall.SIGTERM?)" (SA1016)
- "func envOrDefaultBool is unused" (U1000)
- "should use time.Since instead of time.Now().Sub" (S1012)
2024-03-12 12:14:58 +01:00
macie
d98136bac7 ci: Add quality check
Runs tests and linters after each commit.
2024-03-11 22:19:26 +01:00
macie
c0e2483f6c test: Add basic tests for packet parsing
Tests performed on real-like packets secure expected behavior.
2024-03-11 21:25:34 +01:00
macie
3bd02ed46e refactor: Improve parsing docs
Reveal intentions by:
- extracting magic numbers into constants
- changing function names with >1 responsibilities
- documenting non-obvious behaviors.
2024-03-11 20:35:01 +01:00
Toby
4257788f33 Merge pull request #92 from apernet/wip-doc-openwrt-ipt
docs: instruction for OpenWrt 22.02 or earlier
2024-03-09 09:23:53 -08:00
Haruue
e77c2fabea docs: instruction for OpenWrt 22.02 or earlier 2024-03-09 21:42:49 +08:00
Toby
1dce82745d Merge pull request #85 from apernet/wip-buf
feat: netlink rcv/snd buffer config options
2024-02-29 11:23:08 -08:00
Toby
50cc94889f feat: netlink rcv/snd buffer config options 2024-02-28 17:45:24 -08:00
Toby
5d2d874089 Merge pull request #82 from apernet/update-fet
feat: update FET analyzer to better reflect what's described in the paper
2024-02-26 15:28:33 -08:00
Toby
797dce3dc2 feat: update FET analyzer to better reflect what's described in the paper 2024-02-26 15:27:35 -08:00
Toby
420286a46c Merge pull request #81 from apernet/update-gfwreport
chore: update gfw report links
2024-02-26 15:17:33 -08:00
Toby
531a7b0ceb chore: update gfw report links 2024-02-26 15:17:07 -08:00
Toby
20e0637756 Merge pull request #79 from apernet/update-ci
fix: release workflow
2024-02-26 10:50:44 -08:00
Toby
74dcc92fc6 fix: release workflow 2024-02-26 10:49:19 -08:00
31 changed files with 1843 additions and 1181 deletions

47
.github/workflows/check.yaml vendored Normal file
View File

@@ -0,0 +1,47 @@
name: Quality check
on:
push:
branches:
- "*"
pull_request:
permissions:
contents: read
jobs:
static-analysis:
name: Static analysis
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
- run: go vet ./...
- name: staticcheck
uses: dominikh/staticcheck-action@v1.3.0
with:
install-go: false
tests:
name: Tests
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
- run: go test ./...

View File

@@ -1,6 +1,7 @@
name: Release
on:
release:
types: [ created ]
types: [published]
permissions:
contents: write
@@ -12,15 +13,27 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
goos: [ linux ]
goarch: [ "386", amd64, arm64 ]
goos: [linux]
goarch: ["386", amd64, arm64]
steps:
- uses: actions/checkout@v4
- uses: wangyoucao577/go-release-action@v1
- name: Setup Go
uses: actions/setup-go@v5
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
goversion: "https://go.dev/dl/go1.21.6.linux-amd64.tar.gz"
binary_name: "OpenGFW"
extra_files: LICENSE README.md README.zh.md
go-version: "1.22"
- name: Build
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 0
run: |
mkdir -p build
go build -o build/OpenGFW-${GOOS}-${GOARCH} -ldflags "-s -w" .
- name: Upload
uses: softprops/action-gh-release@v2
with:
files: build/*

3
.gitignore vendored
View File

@@ -205,3 +205,6 @@ $RECYCLE.BIN/
*.lnk
# End of https://www.toptal.com/developers/gitignore/api/windows,macos,linux,go,goland+all,visualstudiocode
# Internal tools not ready for public use yet
tools/flowseq/

View File

@@ -1,5 +1,6 @@
# ![OpenGFW](docs/logo.png)
[![Quality check status](https://github.com/apernet/OpenGFW/actions/workflows/check.yaml/badge.svg)](https://github.com/apernet/OpenGFW/actions/workflows/check.yaml)
[![License][1]][2]
[1]: https://img.shields.io/badge/License-MPL_2.0-brightgreen.svg
@@ -7,20 +8,19 @@
OpenGFW は、あなた専用の DIY 中国のグレートファイアウォール (https://en.wikipedia.org/wiki/Great_Firewall) です。Linux 上で利用可能な柔軟で使いやすいオープンソースプログラムとして提供されています。なぜ権力者だけが楽しむのでしょうか?権力を人々に与え、検閲を民主化する時が来ました。自宅のルーターにサイバー主権のスリルをもたらし、プロのようにフィルタリングを始めましょう - あなたもビッグブラザーになることができます。
**ドキュメントウェブサイト: https://gfw.dev/**
Telegram グループ: https://t.me/OpGFW
> [!CAUTION]
> このプロジェクトはまだ開発の初期段階です。使用は自己責任でお願いします。
> [!NOTE]
> 私たちはこのプロジェクト、特により多くのプロトコル用のアナライザーの実装を手伝ってくれるコントリビューターを探しています!!!
> プロジェクトはまだ初期開発段階にあります。テスト時のリスクは自己責任でお願いします。私たちは、このプロジェクトを一緒に改善するために貢献者を探しています。
## 特徴
- フル IP/TCP 再アセンブル、各種プロトコルアナライザー
- HTTP、TLS、QUIC、DNS、SSH、SOCKS4/5、WireGuard、その他多数
- Shadowsocks の「完全に暗号化されたトラフィック」の検出など (https://gfw.report/publications/usenixsecurity23/data/paper/paper.pdf)
- トロイの木馬キラー (https://github.com/XTLS/Trojan-killer) に基づくトロイの木馬 (プロキシプロトコル) 検出
- HTTP、TLS、QUIC、DNS、SSH、SOCKS4/5、WireGuard、OpenVPN、その他多数
- Shadowsocks、VMess の「完全に暗号化されたトラフィック」の検出など (https://gfw.report/publications/usenixsecurity23/en/)
- Trojan プロキシプロトコル検出
- [WIP] 機械学習に基づくトラフィック分類
- IPv4 と IPv6 をフルサポート
- フローベースのマルチコア負荷分散
@@ -39,119 +39,3 @@ Telegram グループ: https://t.me/OpGFW
- VPN/プロキシサービスの不正利用防止
- トラフィック分析(ログのみモード)
- 独裁的な野心を実現するのを助ける
## 使用方法
### ビルド
```shell
go build
```
### 実行
```shell
export OPENGFW_LOG_LEVEL=debug
./OpenGFW -c config.yaml rules.yaml
```
#### OpenWrt
OpenGFW は OpenWrt 23.05 で動作することがテストされています(他のバージョンも動作するはずですが、検証されていません)。
依存関係をインストールしてください:
```shell
opkg install kmod-nft-queue kmod-nf-conntrack-netlink
```
### 設定例
```yaml
io:
queueSize: 1024
local: true # FORWARD チェーンで OpenGFW を実行したい場合は false に設定する
workers:
count: 4
queueSize: 16
tcpMaxBufferedPagesTotal: 4096
tcpMaxBufferedPagesPerConn: 64
udpMaxStreams: 4096
# 特定のローカルGeoIP / GeoSiteデータベースファイルを読み込むためのパス。
# 設定されていない場合は、https://github.com/LoyalSoldier/v2ray-rules-dat から自動的にダウンロードされます。
# geo:
# geoip: geoip.dat
# geosite: geosite.dat
```
### ルール例
[アナライザーのプロパティ](docs/Analyzers.md)
式言語の構文については、[Expr 言語定義](https://expr-lang.org/docs/language-definition)を参照してください。
```yaml
# ルールは、"action" または "log" の少なくとも一方が設定されていなければなりません。
- name: log horny people
log: true
expr: let sni = string(tls?.req?.sni); sni contains "porn" || sni contains "hentai"
- name: block v2ex http
action: block
expr: string(http?.req?.headers?.host) endsWith "v2ex.com"
- name: block v2ex https
action: block
expr: string(tls?.req?.sni) endsWith "v2ex.com"
- name: block v2ex quic
action: block
expr: string(quic?.req?.sni) endsWith "v2ex.com"
- name: block and log shadowsocks
action: block
log: true
expr: fet != nil && fet.yes
- name: block trojan
action: block
expr: trojan != nil && trojan.yes
- name: v2ex dns poisoning
action: modify
modifier:
name: dns
args:
a: "0.0.0.0"
aaaa: "::"
expr: dns != nil && dns.qr && any(dns.questions, {.name endsWith "v2ex.com"})
- name: block google socks
action: block
expr: string(socks?.req?.addr) endsWith "google.com" && socks?.req?.port == 80
- name: block wireguard by handshake response
action: drop
expr: wireguard?.handshake_response?.receiver_index_matched == true
- name: block bilibili geosite
action: block
expr: geosite(string(tls?.req?.sni), "bilibili")
- name: block CN geoip
action: block
expr: geoip(string(ip.dst), "cn")
- name: block cidr
action: block
expr: cidr(string(ip.dst), "192.168.0.0/16")
```
#### サポートされるアクション
- `allow`: 接続を許可し、それ以上の処理は行わない。
- `block`: 接続をブロックし、それ以上の処理は行わない。
- `drop`: UDP の場合、ルールのトリガーとなったパケットをドロップし、同じフローに含まれる以降のパケットの処理を継続する。TCP の場合は、`block` と同じ。
- `modify`: UDP の場合、与えられた修飾子を使って、ルールをトリガしたパケットを修正し、同じフロー内の今後のパケットを処理し続ける。TCP の場合は、`allow` と同じ。

135
README.md
View File

@@ -1,5 +1,6 @@
# ![OpenGFW](docs/logo.png)
[![Quality check status](https://github.com/apernet/OpenGFW/actions/workflows/check.yaml/badge.svg)](https://github.com/apernet/OpenGFW/actions/workflows/check.yaml)
[![License][1]][2]
[1]: https://img.shields.io/badge/License-MPL_2.0-brightgreen.svg
@@ -10,21 +11,20 @@
OpenGFW is your very own DIY Great Firewall of China (https://en.wikipedia.org/wiki/Great_Firewall), available as a flexible, easy-to-use open source program on Linux. Why let the powers that be have all the fun? It's time to give power to the people and democratize censorship. Bring the thrill of cyber-sovereignty right into your home router and start filtering like a pro - you too can play Big Brother.
**Documentation site: https://gfw.dev/**
Telegram group: https://t.me/OpGFW
> [!CAUTION]
> This project is still in very early stages of development. Use at your own risk.
> [!NOTE]
> We are looking for contributors to help us with this project, especially implementing analyzers for more protocols!!!
> This project is still in very early stages of development. Use at your own risk. We are looking for contributors to help us improve and expand the project.
## Features
- Full IP/TCP reassembly, various protocol analyzers
- HTTP, TLS, QUIC, DNS, SSH, SOCKS4/5, WireGuard, and many more to come
- "Fully encrypted traffic" detection for Shadowsocks,
etc. (https://gfw.report/publications/usenixsecurity23/data/paper/paper.pdf)
- Trojan (proxy protocol) detection based on Trojan-killer (https://github.com/XTLS/Trojan-killer)
- HTTP, TLS, QUIC, DNS, SSH, SOCKS4/5, WireGuard, OpenVPN, and many more to come
- "Fully encrypted traffic" detection for Shadowsocks, VMess,
etc. (https://gfw.report/publications/usenixsecurity23/en/)
- Trojan (proxy protocol) detection
- [WIP] Machine learning based traffic classification
- Full IPv4 and IPv6 support
- Flow-based multicore load balancing
@@ -43,122 +43,3 @@ Telegram group: https://t.me/OpGFW
- Abuse prevention for VPN/proxy services
- Traffic analysis (log only mode)
- Help you fulfill your dictatorial ambitions
## Usage
### Build
```shell
go build
```
### Run
```shell
export OPENGFW_LOG_LEVEL=debug
./OpenGFW -c config.yaml rules.yaml
```
#### OpenWrt
OpenGFW has been tested to work on OpenWrt 23.05 (other versions should also work, just not verified).
Install the dependencies:
```shell
opkg install kmod-nft-queue kmod-nf-conntrack-netlink
```
### Example config
```yaml
io:
queueSize: 1024
local: true # set to false if you want to run OpenGFW on FORWARD chain
workers:
count: 4
queueSize: 16
tcpMaxBufferedPagesTotal: 4096
tcpMaxBufferedPagesPerConn: 64
udpMaxStreams: 4096
# The path to load specific local geoip/geosite db files.
# If not set, they will be automatically downloaded from https://github.com/Loyalsoldier/v2ray-rules-dat
# geo:
# geoip: geoip.dat
# geosite: geosite.dat
```
### Example rules
[Analyzer properties](docs/Analyzers.md)
For syntax of the expression language, please refer
to [Expr Language Definition](https://expr-lang.org/docs/language-definition).
```yaml
# A rule must have at least one of "action" or "log" field set.
- name: log horny people
log: true
expr: let sni = string(tls?.req?.sni); sni contains "porn" || sni contains "hentai"
- name: block v2ex http
action: block
expr: string(http?.req?.headers?.host) endsWith "v2ex.com"
- name: block v2ex https
action: block
expr: string(tls?.req?.sni) endsWith "v2ex.com"
- name: block v2ex quic
action: block
expr: string(quic?.req?.sni) endsWith "v2ex.com"
- name: block and log shadowsocks
action: block
log: true
expr: fet != nil && fet.yes
- name: block trojan
action: block
expr: trojan != nil && trojan.yes
- name: v2ex dns poisoning
action: modify
modifier:
name: dns
args:
a: "0.0.0.0"
aaaa: "::"
expr: dns != nil && dns.qr && any(dns.questions, {.name endsWith "v2ex.com"})
- name: block google socks
action: block
expr: string(socks?.req?.addr) endsWith "google.com" && socks?.req?.port == 80
- name: block wireguard by handshake response
action: drop
expr: wireguard?.handshake_response?.receiver_index_matched == true
- name: block bilibili geosite
action: block
expr: geosite(string(tls?.req?.sni), "bilibili")
- name: block CN geoip
action: block
expr: geoip(string(ip.dst), "cn")
- name: block cidr
action: block
expr: cidr(string(ip.dst), "192.168.0.0/16")
```
#### Supported actions
- `allow`: Allow the connection, no further processing.
- `block`: Block the connection, no further processing.
- `drop`: For UDP, drop the packet that triggered the rule, continue processing future packets in the same flow. For
TCP, same as `block`.
- `modify`: For UDP, modify the packet that triggered the rule using the given modifier, continue processing future
packets in the same flow. For TCP, same as `allow`.

View File

@@ -1,5 +1,6 @@
# ![OpenGFW](docs/logo.png)
[![Quality check status](https://github.com/apernet/OpenGFW/actions/workflows/check.yaml/badge.svg)](https://github.com/apernet/OpenGFW/actions/workflows/check.yaml)
[![License][1]][2]
[1]: https://img.shields.io/badge/License-MPL_2.0-brightgreen.svg
@@ -7,20 +8,19 @@
OpenGFW 是一个 Linux 上灵活、易用、开源的 DIY [GFW](https://zh.wikipedia.org/wiki/%E9%98%B2%E7%81%AB%E9%95%BF%E5%9F%8E) 实现,并且在许多方面比真正的 GFW 更强大。为何让那些掌权者独享乐趣?是时候把权力归还给人民,人人有墙建了。立即安装可以部署在家用路由器上的网络主权 - 你也能是老大哥。
**文档网站: https://gfw.dev/**
Telegram 群组: https://t.me/OpGFW
> [!CAUTION]
> 本项目仍处于早期开发阶段。测试时自行承担风险。
> [!NOTE]
> 我们正在寻求贡献者一起完善本项目,尤其是实现更多协议的解析器!
> 本项目仍处于早期开发阶段。测试时自行承担风险。我们正在寻求贡献者一起完善本项目。
## 功能
- 完整的 IP/TCP 重组,各种协议解析器
- HTTP, TLS, QUIC, DNS, SSH, SOCKS4/5, WireGuard, 更多协议正在开发中
- Shadowsocks 等 "全加密流量" 检测 (https://gfw.report/publications/usenixsecurity23/data/paper/paper.pdf)
- 基于 Trojan-killer 的 Trojan 检测 (https://github.com/XTLS/Trojan-killer)
- HTTP, TLS, QUIC, DNS, SSH, SOCKS4/5, WireGuard, OpenVPN, 更多协议正在开发中
- Shadowsocks, VMess 等 "全加密流量" 检测 (https://gfw.report/publications/usenixsecurity23/zh/)
- Trojan 协议检测
- [开发中] 基于机器学习的流量分类
- 同等支持 IPv4 和 IPv6
- 基于流的多核负载均衡
@@ -39,119 +39,3 @@ Telegram 群组: https://t.me/OpGFW
- VPN/代理服务滥用防护
- 流量分析 (纯日志模式)
- 助你实现你的独裁野心
## 使用
### 构建
```shell
go build
```
### 运行
```shell
export OPENGFW_LOG_LEVEL=debug
./OpenGFW -c config.yaml rules.yaml
```
#### OpenWrt
OpenGFW 在 OpenWrt 23.05 上测试可用(其他版本应该也可以,暂时未经验证)。
安装依赖:
```shell
opkg install kmod-nft-queue kmod-nf-conntrack-netlink
```
### 样例配置
```yaml
io:
queueSize: 1024
local: true # 如果需要在 FORWARD 链上运行 OpenGFW请设置为 false
workers:
count: 4
queueSize: 16
tcpMaxBufferedPagesTotal: 4096
tcpMaxBufferedPagesPerConn: 64
udpMaxStreams: 4096
# 指定的 geoip/geosite 档案路径
# 如果未设置,将自动从 https://github.com/Loyalsoldier/v2ray-rules-dat 下载
# geo:
# geoip: geoip.dat
# geosite: geosite.dat
```
### 样例规则
[解析器属性](docs/Analyzers.md)
规则的语法请参考 [Expr Language Definition](https://expr-lang.org/docs/language-definition)。
```yaml
# 每条规则必须至少包含 action 或 log 中的一个。
- name: log horny people
log: true
expr: let sni = string(tls?.req?.sni); sni contains "porn" || sni contains "hentai"
- name: block v2ex http
action: block
expr: string(http?.req?.headers?.host) endsWith "v2ex.com"
- name: block v2ex https
action: block
expr: string(tls?.req?.sni) endsWith "v2ex.com"
- name: block v2ex quic
action: block
expr: string(quic?.req?.sni) endsWith "v2ex.com"
- name: block and log shadowsocks
action: block
log: true
expr: fet != nil && fet.yes
- name: block trojan
action: block
expr: trojan != nil && trojan.yes
- name: v2ex dns poisoning
action: modify
modifier:
name: dns
args:
a: "0.0.0.0"
aaaa: "::"
expr: dns != nil && dns.qr && any(dns.questions, {.name endsWith "v2ex.com"})
- name: block google socks
action: block
expr: string(socks?.req?.addr) endsWith "google.com" && socks?.req?.port == 80
- name: block wireguard by handshake response
action: drop
expr: wireguard?.handshake_response?.receiver_index_matched == true
- name: block bilibili geosite
action: block
expr: geosite(string(tls?.req?.sni), "bilibili")
- name: block CN geoip
action: block
expr: geoip(string(ip.dst), "cn")
- name: block cidr
action: block
expr: cidr(string(ip.dst), "192.168.0.0/16")
```
#### 支持的 action
- `allow`: 放行连接,不再处理后续的包。
- `block`: 阻断连接,不再处理后续的包。
- `drop`: 对于 UDP丢弃触发规则的包但继续处理同一流中的后续包。对于 TCP效果同 `block`
- `modify`: 对于 UDP用指定的修改器修改触发规则的包然后继续处理同一流中的后续包。对于 TCP效果同 `allow`

View File

@@ -5,7 +5,26 @@ import (
"github.com/apernet/OpenGFW/analyzer/utils"
)
func ParseTLSClientHello(chBuf *utils.ByteBuffer) analyzer.PropMap {
// TLS record types.
const (
RecordTypeHandshake = 0x16
)
// TLS handshake message types.
const (
TypeClientHello = 0x01
TypeServerHello = 0x02
)
// TLS extension numbers.
const (
extServerName = 0x0000
extALPN = 0x0010
extSupportedVersions = 0x002b
extEncryptedClientHello = 0xfe0d
)
func ParseTLSClientHelloMsgData(chBuf *utils.ByteBuffer) analyzer.PropMap {
var ok bool
m := make(analyzer.PropMap)
// Version, random & session ID length combined are within 35 bytes,
@@ -76,7 +95,7 @@ func ParseTLSClientHello(chBuf *utils.ByteBuffer) analyzer.PropMap {
return m
}
func ParseTLSServerHello(shBuf *utils.ByteBuffer) analyzer.PropMap {
func ParseTLSServerHelloMsgData(shBuf *utils.ByteBuffer) analyzer.PropMap {
var ok bool
m := make(analyzer.PropMap)
// Version, random & session ID length combined are within 35 bytes,
@@ -133,7 +152,7 @@ func ParseTLSServerHello(shBuf *utils.ByteBuffer) analyzer.PropMap {
func parseTLSExtensions(extType uint16, extDataBuf *utils.ByteBuffer, m analyzer.PropMap) bool {
switch extType {
case 0x0000: // SNI
case extServerName:
ok := extDataBuf.Skip(2) // Ignore list length, we only care about the first entry for now
if !ok {
// Not enough data for list length
@@ -154,7 +173,7 @@ func parseTLSExtensions(extType uint16, extDataBuf *utils.ByteBuffer, m analyzer
// Not enough data for SNI
return false
}
case 0x0010: // ALPN
case extALPN:
ok := extDataBuf.Skip(2) // Ignore list length, as we read until the end
if !ok {
// Not enough data for list length
@@ -175,7 +194,7 @@ func parseTLSExtensions(extType uint16, extDataBuf *utils.ByteBuffer, m analyzer
alpnList = append(alpnList, alpn)
}
m["alpn"] = alpnList
case 0x002b: // Supported Versions
case extSupportedVersions:
if extDataBuf.Len() == 2 {
// Server only selects one version
m["supported_versions"], _ = extDataBuf.GetUint16(false, true)
@@ -197,7 +216,7 @@ func parseTLSExtensions(extType uint16, extDataBuf *utils.ByteBuffer, m analyzer
}
m["supported_versions"] = versions
}
case 0xfe0d: // ECH
case extEncryptedClientHello:
// We can't parse ECH for now, just set a flag
m["ech"] = true
}

View File

@@ -143,8 +143,11 @@ func isTLSorHTTP(bytes []byte) bool {
if len(bytes) < 3 {
return false
}
if bytes[0] == 0x16 && bytes[1] == 0x03 && bytes[2] <= 0x03 {
// TLS handshake for TLS 1.0-1.3
// "We observe that the GFW exempts any connection whose first
// three bytes match the following regular expression:
// [\x16-\x17]\x03[\x00-\x09]" - from the paper in Section 4.3
if bytes[0] >= 0x16 && bytes[0] <= 0x17 &&
bytes[1] == 0x03 && bytes[2] <= 0x09 {
return true
}
// HTTP request

64
analyzer/tcp/http_test.go Normal file
View File

@@ -0,0 +1,64 @@
package tcp
import (
"reflect"
"strings"
"testing"
"github.com/apernet/OpenGFW/analyzer"
)
func TestHTTPParsing_Request(t *testing.T) {
testCases := map[string]analyzer.PropMap{
"GET / HTTP/1.1\r\n": {
"method": "GET", "path": "/", "version": "HTTP/1.1",
},
"POST /hello?a=1&b=2 HTTP/1.0\r\n": {
"method": "POST", "path": "/hello?a=1&b=2", "version": "HTTP/1.0",
},
"PUT /world HTTP/1.1\r\nContent-Length: 4\r\n\r\nbody": {
"method": "PUT", "path": "/world", "version": "HTTP/1.1", "headers": analyzer.PropMap{"content-length": "4"},
},
"DELETE /goodbye HTTP/2.0\r\n": {
"method": "DELETE", "path": "/goodbye", "version": "HTTP/2.0",
},
}
for tc, want := range testCases {
t.Run(strings.Split(tc, " ")[0], func(t *testing.T) {
tc, want := tc, want
t.Parallel()
u, _ := newHTTPStream(nil).Feed(false, false, false, 0, []byte(tc))
got := u.M.Get("req")
if !reflect.DeepEqual(got, want) {
t.Errorf("\"%s\" parsed = %v, want %v", tc, got, want)
}
})
}
}
func TestHTTPParsing_Response(t *testing.T) {
testCases := map[string]analyzer.PropMap{
"HTTP/1.0 200 OK\r\nContent-Length: 4\r\n\r\nbody": {
"version": "HTTP/1.0", "status": 200,
"headers": analyzer.PropMap{"content-length": "4"},
},
"HTTP/2.0 204 No Content\r\n\r\n": {
"version": "HTTP/2.0", "status": 204,
},
}
for tc, want := range testCases {
t.Run(strings.Split(tc, " ")[0], func(t *testing.T) {
tc, want := tc, want
t.Parallel()
u, _ := newHTTPStream(nil).Feed(true, false, false, 0, []byte(tc))
got := u.M.Get("resp")
if !reflect.DeepEqual(got, want) {
t.Errorf("\"%s\" parsed = %v, want %v", tc, got, want)
}
})
}
}

View File

@@ -208,10 +208,10 @@ func (s *socksStream) parseSocks5ReqMethod() utils.LSMAction {
switch method {
case Socks5AuthNotRequired:
s.authReqMethod = Socks5AuthNotRequired
break
return utils.LSMActionNext
case Socks5AuthPassword:
s.authReqMethod = Socks5AuthPassword
break
return utils.LSMActionNext
default:
// TODO: more auth method to support
}

View File

@@ -44,12 +44,12 @@ type tlsStream struct {
func newTLSStream(logger analyzer.Logger) *tlsStream {
s := &tlsStream{logger: logger, reqBuf: &utils.ByteBuffer{}, respBuf: &utils.ByteBuffer{}}
s.reqLSM = utils.NewLinearStateMachine(
s.tlsClientHelloSanityCheck,
s.parseClientHello,
s.tlsClientHelloPreprocess,
s.parseClientHelloData,
)
s.respLSM = utils.NewLinearStateMachine(
s.tlsServerHelloSanityCheck,
s.parseServerHello,
s.tlsServerHelloPreprocess,
s.parseServerHelloData,
)
return s
}
@@ -89,61 +89,105 @@ func (s *tlsStream) Feed(rev, start, end bool, skip int, data []byte) (u *analyz
return update, cancelled || (s.reqDone && s.respDone)
}
func (s *tlsStream) tlsClientHelloSanityCheck() utils.LSMAction {
data, ok := s.reqBuf.Get(9, true)
// tlsClientHelloPreprocess validates ClientHello message.
//
// During validation, message header and first handshake header may be removed
// from `s.reqBuf`.
func (s *tlsStream) tlsClientHelloPreprocess() utils.LSMAction {
// headers size: content type (1 byte) + legacy protocol version (2 bytes) +
// + content length (2 bytes) + message type (1 byte) +
// + handshake length (3 bytes)
const headersSize = 9
// minimal data size: protocol version (2 bytes) + random (32 bytes) +
// + session ID (1 byte) + cipher suites (4 bytes) +
// + compression methods (2 bytes) + no extensions
const minDataSize = 41
header, ok := s.reqBuf.Get(headersSize, true)
if !ok {
// not a full header yet
return utils.LSMActionPause
}
if data[0] != 0x16 || data[5] != 0x01 {
// Not a TLS handshake, or not a client hello
if header[0] != internal.RecordTypeHandshake || header[5] != internal.TypeClientHello {
return utils.LSMActionCancel
}
s.clientHelloLen = int(data[6])<<16 | int(data[7])<<8 | int(data[8])
if s.clientHelloLen < 41 {
// 2 (Protocol Version) +
// 32 (Random) +
// 1 (Session ID Length) +
// 2 (Cipher Suites Length) +_ws.col.protocol == "TLSv1.3"
// 2 (Cipher Suite) +
// 1 (Compression Methods Length) +
// 1 (Compression Method) +
// No extensions
// This should be the bare minimum for a client hello
s.clientHelloLen = int(header[6])<<16 | int(header[7])<<8 | int(header[8])
if s.clientHelloLen < minDataSize {
return utils.LSMActionCancel
}
// TODO: something is missing. See:
// const messageHeaderSize = 4
// fullMessageLen := int(header[3])<<8 | int(header[4])
// msgNo := fullMessageLen / int(messageHeaderSize+s.serverHelloLen)
// if msgNo != 1 {
// // what here?
// }
// if messageNo != int(messageNo) {
// // what here?
// }
return utils.LSMActionNext
}
func (s *tlsStream) tlsServerHelloSanityCheck() utils.LSMAction {
data, ok := s.respBuf.Get(9, true)
// tlsServerHelloPreprocess validates ServerHello message.
//
// During validation, message header and first handshake header may be removed
// from `s.reqBuf`.
func (s *tlsStream) tlsServerHelloPreprocess() utils.LSMAction {
// header size: content type (1 byte) + legacy protocol version (2 byte) +
// + content length (2 byte) + message type (1 byte) +
// + handshake length (3 byte)
const headersSize = 9
// minimal data size: server version (2 byte) + random (32 byte) +
// + session ID (>=1 byte) + cipher suite (2 byte) +
// + compression method (1 byte) + no extensions
const minDataSize = 38
header, ok := s.respBuf.Get(headersSize, true)
if !ok {
// not a full header yet
return utils.LSMActionPause
}
if data[0] != 0x16 || data[5] != 0x02 {
// Not a TLS handshake, or not a server hello
if header[0] != internal.RecordTypeHandshake || header[5] != internal.TypeServerHello {
return utils.LSMActionCancel
}
s.serverHelloLen = int(data[6])<<16 | int(data[7])<<8 | int(data[8])
if s.serverHelloLen < 38 {
// 2 (Protocol Version) +
// 32 (Random) +
// 1 (Session ID Length) +
// 2 (Cipher Suite) +
// 1 (Compression Method) +
// No extensions
// This should be the bare minimum for a server hello
s.serverHelloLen = int(header[6])<<16 | int(header[7])<<8 | int(header[8])
if s.serverHelloLen < minDataSize {
return utils.LSMActionCancel
}
// TODO: something is missing. See example:
// const messageHeaderSize = 4
// fullMessageLen := int(header[3])<<8 | int(header[4])
// msgNo := fullMessageLen / int(messageHeaderSize+s.serverHelloLen)
// if msgNo != 1 {
// // what here?
// }
// if messageNo != int(messageNo) {
// // what here?
// }
return utils.LSMActionNext
}
func (s *tlsStream) parseClientHello() utils.LSMAction {
// parseClientHelloData converts valid ClientHello message data (without
// headers) into `analyzer.PropMap`.
//
// Parsing error may leave `s.reqBuf` in an unusable state.
func (s *tlsStream) parseClientHelloData() utils.LSMAction {
chBuf, ok := s.reqBuf.GetSubBuffer(s.clientHelloLen, true)
if !ok {
// Not a full client hello yet
return utils.LSMActionPause
}
m := internal.ParseTLSClientHello(chBuf)
m := internal.ParseTLSClientHelloMsgData(chBuf)
if m == nil {
return utils.LSMActionCancel
} else {
@@ -153,13 +197,17 @@ func (s *tlsStream) parseClientHello() utils.LSMAction {
}
}
func (s *tlsStream) parseServerHello() utils.LSMAction {
// parseServerHelloData converts valid ServerHello message data (without
// headers) into `analyzer.PropMap`.
//
// Parsing error may leave `s.respBuf` in an unusable state.
func (s *tlsStream) parseServerHelloData() utils.LSMAction {
shBuf, ok := s.respBuf.GetSubBuffer(s.serverHelloLen, true)
if !ok {
// Not a full server hello yet
return utils.LSMActionPause
}
m := internal.ParseTLSServerHello(shBuf)
m := internal.ParseTLSServerHelloMsgData(shBuf)
if m == nil {
return utils.LSMActionCancel
} else {

69
analyzer/tcp/tls_test.go Normal file
View File

@@ -0,0 +1,69 @@
package tcp
import (
"reflect"
"testing"
"github.com/apernet/OpenGFW/analyzer"
)
func TestTlsStreamParsing_ClientHello(t *testing.T) {
// example packet taken from <https://tls12.xargs.org/#client-hello/annotated>
clientHello := []byte{
0x16, 0x03, 0x01, 0x00, 0xa5, 0x01, 0x00, 0x00, 0xa1, 0x03, 0x03, 0x00,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c,
0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x20, 0xcc, 0xa8,
0xcc, 0xa9, 0xc0, 0x2f, 0xc0, 0x30, 0xc0, 0x2b, 0xc0, 0x2c, 0xc0, 0x13,
0xc0, 0x09, 0xc0, 0x14, 0xc0, 0x0a, 0x00, 0x9c, 0x00, 0x9d, 0x00, 0x2f,
0x00, 0x35, 0xc0, 0x12, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x58, 0x00, 0x00,
0x00, 0x18, 0x00, 0x16, 0x00, 0x00, 0x13, 0x65, 0x78, 0x61, 0x6d, 0x70,
0x6c, 0x65, 0x2e, 0x75, 0x6c, 0x66, 0x68, 0x65, 0x69, 0x6d, 0x2e, 0x6e,
0x65, 0x74, 0x00, 0x05, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
0x0a, 0x00, 0x0a, 0x00, 0x08, 0x00, 0x1d, 0x00, 0x17, 0x00, 0x18, 0x00,
0x19, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, 0x00, 0x0d, 0x00, 0x12, 0x00,
0x10, 0x04, 0x01, 0x04, 0x03, 0x05, 0x01, 0x05, 0x03, 0x06, 0x01, 0x06,
0x03, 0x02, 0x01, 0x02, 0x03, 0xff, 0x01, 0x00, 0x01, 0x00, 0x00, 0x12,
0x00, 0x00,
}
want := analyzer.PropMap{
"ciphers": []uint16{52392, 52393, 49199, 49200, 49195, 49196, 49171, 49161, 49172, 49162, 156, 157, 47, 53, 49170, 10},
"compression": []uint8{0},
"random": []uint8{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31},
"session": []uint8{},
"sni": "example.ulfheim.net",
"version": uint16(771),
}
s := newTLSStream(nil)
u, _ := s.Feed(false, false, false, 0, clientHello)
got := u.M.Get("req")
if !reflect.DeepEqual(got, want) {
t.Errorf("%d B parsed = %v, want %v", len(clientHello), got, want)
}
}
func TestTlsStreamParsing_ServerHello(t *testing.T) {
// example packet taken from <https://tls12.xargs.org/#server-hello/annotated>
serverHello := []byte{
0x16, 0x03, 0x03, 0x00, 0x31, 0x02, 0x00, 0x00, 0x2d, 0x03, 0x03, 0x70,
0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c,
0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x00, 0xc0, 0x13, 0x00, 0x00,
0x05, 0xff, 0x01, 0x00, 0x01, 0x00,
}
want := analyzer.PropMap{
"cipher": uint16(49171),
"compression": uint8(0),
"random": []uint8{112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143},
"session": []uint8{},
"version": uint16(771),
}
s := newTLSStream(nil)
u, _ := s.Feed(true, false, false, 0, serverHello)
got := u.M.Get("resp")
if !reflect.DeepEqual(got, want) {
t.Errorf("%d B parsed = %v, want %v", len(serverHello), got, want)
}
}

View File

@@ -9,22 +9,15 @@ import (
var _ analyzer.TCPAnalyzer = (*TrojanAnalyzer)(nil)
// CCS stands for "Change Cipher Spec"
var trojanCCS = []byte{20, 3, 3, 0, 1, 1}
var ccsPattern = []byte{20, 3, 3, 0, 1, 1}
const (
trojanUpLB = 650
trojanUpUB = 1000
trojanDownLB1 = 170
trojanDownUB1 = 180
trojanDownLB2 = 3000
trojanDownUB2 = 7500
)
// TrojanAnalyzer uses a very simple packet length based check to determine
// if a TLS connection is actually the Trojan proxy protocol.
// The algorithm is from the following project, with small modifications:
// https://github.com/XTLS/Trojan-killer
// Warning: Experimental only. This method is known to have significant false positives and false negatives.
// TrojanAnalyzer uses length-based heuristics to detect Trojan traffic based on
// its "TLS-in-TLS" nature. The heuristics are trained using a decision tree with
// about 20k Trojan samples and 30k non-Trojan samples. The tree is then converted
// to code using a custom tool and inlined here (isTrojanSeq function).
// Accuracy: 1% false positive rate, 10% false negative rate.
// We do NOT recommend directly blocking all positive connections, as this may
// break legitimate TLS connections.
type TrojanAnalyzer struct{}
func (a *TrojanAnalyzer) Name() string {
@@ -32,7 +25,7 @@ func (a *TrojanAnalyzer) Name() string {
}
func (a *TrojanAnalyzer) Limit() int {
return 16384
return 512000
}
func (a *TrojanAnalyzer) NewTCP(info analyzer.TCPInfo, logger analyzer.Logger) analyzer.TCPStream {
@@ -40,10 +33,12 @@ func (a *TrojanAnalyzer) NewTCP(info analyzer.TCPInfo, logger analyzer.Logger) a
}
type trojanStream struct {
logger analyzer.Logger
active bool
upCount int
downCount int
logger analyzer.Logger
first bool
count bool
rev bool
seq [4]int
seqIndex int
}
func newTrojanStream(logger analyzer.Logger) *trojanStream {
@@ -57,35 +52,466 @@ func (s *trojanStream) Feed(rev, start, end bool, skip int, data []byte) (u *ana
if len(data) == 0 {
return nil, false
}
if !rev && !s.active && len(data) >= 6 && bytes.Equal(data[:6], trojanCCS) {
// Client CCS encountered, start counting
s.active = true
if s.first {
s.first = false
// Stop if it's not a valid TLS connection
if !(!rev && len(data) >= 3 && data[0] >= 0x16 && data[0] <= 0x17 &&
data[1] == 0x03 && data[2] <= 0x09) {
return nil, true
}
}
if s.active {
if rev {
// Down direction
s.downCount += len(data)
if !rev && !s.count && len(data) >= 6 && bytes.Equal(data[:6], ccsPattern) {
// Client Change Cipher Spec encountered, start counting
s.count = true
}
if s.count {
if rev == s.rev {
// Same direction as last time, just update the number
s.seq[s.seqIndex] += len(data)
} else {
// Up direction
if s.upCount >= trojanUpLB && s.upCount <= trojanUpUB &&
((s.downCount >= trojanDownLB1 && s.downCount <= trojanDownUB1) ||
(s.downCount >= trojanDownLB2 && s.downCount <= trojanDownUB2)) {
// Different direction, bump the index
s.seqIndex += 1
if s.seqIndex == 4 {
return &analyzer.PropUpdate{
Type: analyzer.PropUpdateReplace,
M: analyzer.PropMap{
"up": s.upCount,
"down": s.downCount,
"yes": true,
"seq": s.seq,
"yes": isTrojanSeq(s.seq),
},
}, true
}
s.upCount += len(data)
s.seq[s.seqIndex] += len(data)
s.rev = rev
}
}
// Give up when either direction is over the limit
return nil, s.upCount > trojanUpUB || s.downCount > trojanDownUB2
return nil, false
}
func (s *trojanStream) Close(limited bool) *analyzer.PropUpdate {
return nil
}
func isTrojanSeq(seq [4]int) bool {
length1 := seq[0]
length2 := seq[1]
length3 := seq[2]
length4 := seq[3]
if length2 <= 2431 {
if length2 <= 157 {
if length1 <= 156 {
if length3 <= 108 {
return false
} else {
return false
}
} else {
if length1 <= 892 {
if length3 <= 40 {
return false
} else {
if length3 <= 788 {
if length4 <= 185 {
if length1 <= 411 {
return true
} else {
return false
}
} else {
if length2 <= 112 {
return false
} else {
return true
}
}
} else {
if length3 <= 1346 {
if length1 <= 418 {
return false
} else {
return true
}
} else {
return false
}
}
}
} else {
if length2 <= 120 {
if length2 <= 63 {
return false
} else {
if length4 <= 653 {
return false
} else {
return false
}
}
} else {
return false
}
}
}
} else {
if length1 <= 206 {
if length1 <= 185 {
if length1 <= 171 {
return false
} else {
if length4 <= 211 {
return false
} else {
return false
}
}
} else {
if length2 <= 251 {
return true
} else {
return false
}
}
} else {
if length2 <= 286 {
if length1 <= 1123 {
if length3 <= 70 {
return false
} else {
if length1 <= 659 {
if length3 <= 370 {
return true
} else {
return false
}
} else {
if length4 <= 272 {
return false
} else {
return true
}
}
}
} else {
if length4 <= 537 {
if length2 <= 276 {
if length3 <= 1877 {
return false
} else {
return false
}
} else {
return false
}
} else {
if length1 <= 1466 {
if length1 <= 1435 {
return false
} else {
return true
}
} else {
if length2 <= 193 {
return false
} else {
return false
}
}
}
}
} else {
if length1 <= 284 {
if length1 <= 277 {
if length2 <= 726 {
return false
} else {
if length2 <= 768 {
return true
} else {
return false
}
}
} else {
if length2 <= 782 {
if length4 <= 783 {
return true
} else {
return false
}
} else {
return false
}
}
} else {
if length2 <= 492 {
if length2 <= 396 {
if length2 <= 322 {
return false
} else {
return false
}
} else {
if length4 <= 971 {
return false
} else {
return true
}
}
} else {
if length2 <= 2128 {
if length2 <= 1418 {
return false
} else {
return false
}
} else {
if length3 <= 103 {
return false
} else {
return false
}
}
}
}
}
}
}
} else {
if length2 <= 6232 {
if length3 <= 85 {
if length2 <= 3599 {
return false
} else {
if length1 <= 613 {
return false
} else {
return false
}
}
} else {
if length3 <= 220 {
if length4 <= 1173 {
if length1 <= 874 {
if length4 <= 337 {
if length4 <= 68 {
return true
} else {
return true
}
} else {
if length1 <= 667 {
return true
} else {
return true
}
}
} else {
if length3 <= 108 {
if length1 <= 1930 {
return true
} else {
return true
}
} else {
if length2 <= 5383 {
return false
} else {
return true
}
}
}
} else {
return false
}
} else {
if length1 <= 664 {
if length3 <= 411 {
if length3 <= 383 {
if length4 <= 346 {
return true
} else {
return false
}
} else {
if length1 <= 445 {
return true
} else {
return false
}
}
} else {
if length2 <= 3708 {
if length4 <= 307 {
return true
} else {
return false
}
} else {
if length2 <= 4656 {
return false
} else {
return false
}
}
}
} else {
if length1 <= 1055 {
if length3 <= 580 {
if length1 <= 724 {
return true
} else {
return false
}
} else {
if length1 <= 678 {
return false
} else {
return true
}
}
} else {
if length2 <= 5352 {
if length3 <= 1586 {
return false
} else {
return false
}
} else {
if length4 <= 2173 {
return true
} else {
return false
}
}
}
}
}
}
} else {
if length2 <= 9408 {
if length1 <= 670 {
if length4 <= 76 {
if length3 <= 175 {
return true
} else {
return true
}
} else {
if length2 <= 9072 {
if length3 <= 314 {
if length3 <= 179 {
return false
} else {
return false
}
} else {
if length4 <= 708 {
return false
} else {
return false
}
}
} else {
return true
}
}
} else {
if length1 <= 795 {
if length2 <= 6334 {
if length2 <= 6288 {
return true
} else {
return false
}
} else {
if length4 <= 6404 {
if length2 <= 8194 {
return true
} else {
return true
}
} else {
if length2 <= 8924 {
return false
} else {
return true
}
}
}
} else {
if length3 <= 732 {
if length1 <= 1397 {
if length3 <= 179 {
return false
} else {
return false
}
} else {
if length1 <= 1976 {
return false
} else {
return false
}
}
} else {
if length1 <= 2840 {
if length1 <= 2591 {
return false
} else {
return true
}
} else {
return false
}
}
}
}
} else {
if length4 <= 30 {
return false
} else {
if length2 <= 13314 {
if length4 <= 1786 {
if length2 <= 13018 {
if length4 <= 869 {
return false
} else {
return false
}
} else {
return true
}
} else {
if length3 <= 775 {
return false
} else {
return false
}
}
} else {
if length4 <= 73 {
return false
} else {
if length3 <= 640 {
if length3 <= 237 {
return false
} else {
return false
}
} else {
if length2 <= 43804 {
return false
} else {
return false
}
}
}
}
}
}
}
}
}

384
analyzer/udp/openvpn.go Normal file
View File

@@ -0,0 +1,384 @@
package udp
import (
"github.com/apernet/OpenGFW/analyzer"
"github.com/apernet/OpenGFW/analyzer/utils"
)
var (
_ analyzer.UDPAnalyzer = (*OpenVPNAnalyzer)(nil)
_ analyzer.TCPAnalyzer = (*OpenVPNAnalyzer)(nil)
)
var (
_ analyzer.UDPStream = (*openvpnUDPStream)(nil)
_ analyzer.TCPStream = (*openvpnTCPStream)(nil)
)
// Ref paper:
// https://www.usenix.org/system/files/sec22fall_xue-diwen.pdf
// OpenVPN Opcodes definitions from:
// https://github.com/OpenVPN/openvpn/blob/master/src/openvpn/ssl_pkt.h
const (
OpenVPNControlHardResetClientV1 = 1
OpenVPNControlHardResetServerV1 = 2
OpenVPNControlSoftResetV1 = 3
OpenVPNControlV1 = 4
OpenVPNAckV1 = 5
OpenVPNDataV1 = 6
OpenVPNControlHardResetClientV2 = 7
OpenVPNControlHardResetServerV2 = 8
OpenVPNDataV2 = 9
OpenVPNControlHardResetClientV3 = 10
OpenVPNControlWkcV1 = 11
)
const (
OpenVPNMinPktLen = 6
OpenVPNTCPPktDefaultLimit = 256
OpenVPNUDPPktDefaultLimit = 256
)
type OpenVPNAnalyzer struct{}
func (a *OpenVPNAnalyzer) Name() string {
return "openvpn"
}
func (a *OpenVPNAnalyzer) Limit() int {
return 0
}
func (a *OpenVPNAnalyzer) NewUDP(info analyzer.UDPInfo, logger analyzer.Logger) analyzer.UDPStream {
return newOpenVPNUDPStream(logger)
}
func (a *OpenVPNAnalyzer) NewTCP(info analyzer.TCPInfo, logger analyzer.Logger) analyzer.TCPStream {
return newOpenVPNTCPStream(logger)
}
type openvpnPkt struct {
pktLen uint16 // 16 bits, TCP proto only
opcode byte // 5 bits
_keyId byte // 3 bits, not used
// We don't care about the rest of the packet
// payload []byte
}
type openvpnStream struct {
logger analyzer.Logger
reqUpdated bool
reqLSM *utils.LinearStateMachine
reqDone bool
respUpdated bool
respLSM *utils.LinearStateMachine
respDone bool
rxPktCnt int
txPktCnt int
pktLimit int
reqPktParse func() (*openvpnPkt, utils.LSMAction)
respPktParse func() (*openvpnPkt, utils.LSMAction)
lastOpcode byte
}
func (o *openvpnStream) parseCtlHardResetClient() utils.LSMAction {
pkt, action := o.reqPktParse()
if action != utils.LSMActionNext {
return action
}
if pkt.opcode != OpenVPNControlHardResetClientV1 &&
pkt.opcode != OpenVPNControlHardResetClientV2 &&
pkt.opcode != OpenVPNControlHardResetClientV3 {
return utils.LSMActionCancel
}
o.lastOpcode = pkt.opcode
return utils.LSMActionNext
}
func (o *openvpnStream) parseCtlHardResetServer() utils.LSMAction {
if o.lastOpcode != OpenVPNControlHardResetClientV1 &&
o.lastOpcode != OpenVPNControlHardResetClientV2 &&
o.lastOpcode != OpenVPNControlHardResetClientV3 {
return utils.LSMActionCancel
}
pkt, action := o.respPktParse()
if action != utils.LSMActionNext {
return action
}
if pkt.opcode != OpenVPNControlHardResetServerV1 &&
pkt.opcode != OpenVPNControlHardResetServerV2 {
return utils.LSMActionCancel
}
o.lastOpcode = pkt.opcode
return utils.LSMActionNext
}
func (o *openvpnStream) parseReq() utils.LSMAction {
pkt, action := o.reqPktParse()
if action != utils.LSMActionNext {
return action
}
if pkt.opcode != OpenVPNControlSoftResetV1 &&
pkt.opcode != OpenVPNControlV1 &&
pkt.opcode != OpenVPNAckV1 &&
pkt.opcode != OpenVPNDataV1 &&
pkt.opcode != OpenVPNDataV2 &&
pkt.opcode != OpenVPNControlWkcV1 {
return utils.LSMActionCancel
}
o.txPktCnt += 1
o.reqUpdated = true
return utils.LSMActionPause
}
func (o *openvpnStream) parseResp() utils.LSMAction {
pkt, action := o.respPktParse()
if action != utils.LSMActionNext {
return action
}
if pkt.opcode != OpenVPNControlSoftResetV1 &&
pkt.opcode != OpenVPNControlV1 &&
pkt.opcode != OpenVPNAckV1 &&
pkt.opcode != OpenVPNDataV1 &&
pkt.opcode != OpenVPNDataV2 &&
pkt.opcode != OpenVPNControlWkcV1 {
return utils.LSMActionCancel
}
o.rxPktCnt += 1
o.respUpdated = true
return utils.LSMActionPause
}
type openvpnUDPStream struct {
openvpnStream
curPkt []byte
// We don't introduce `invalidCount` here to decrease the false positive rate
// invalidCount int
}
func newOpenVPNUDPStream(logger analyzer.Logger) *openvpnUDPStream {
s := &openvpnUDPStream{
openvpnStream: openvpnStream{
logger: logger,
pktLimit: OpenVPNUDPPktDefaultLimit,
},
}
s.respPktParse = s.parsePkt
s.reqPktParse = s.parsePkt
s.reqLSM = utils.NewLinearStateMachine(
s.parseCtlHardResetClient,
s.parseReq,
)
s.respLSM = utils.NewLinearStateMachine(
s.parseCtlHardResetServer,
s.parseResp,
)
return s
}
func (o *openvpnUDPStream) Feed(rev bool, data []byte) (u *analyzer.PropUpdate, d bool) {
if len(data) == 0 {
return nil, false
}
var update *analyzer.PropUpdate
var cancelled bool
o.curPkt = data
if rev {
o.respUpdated = false
cancelled, o.respDone = o.respLSM.Run()
if o.respUpdated {
update = &analyzer.PropUpdate{
Type: analyzer.PropUpdateReplace,
M: analyzer.PropMap{"rx_pkt_cnt": o.rxPktCnt, "tx_pkt_cnt": o.txPktCnt},
}
o.respUpdated = false
}
} else {
o.reqUpdated = false
cancelled, o.reqDone = o.reqLSM.Run()
if o.reqUpdated {
update = &analyzer.PropUpdate{
Type: analyzer.PropUpdateReplace,
M: analyzer.PropMap{"rx_pkt_cnt": o.rxPktCnt, "tx_pkt_cnt": o.txPktCnt},
}
o.reqUpdated = false
}
}
return update, cancelled || (o.reqDone && o.respDone) || o.rxPktCnt+o.txPktCnt > o.pktLimit
}
func (o *openvpnUDPStream) Close(limited bool) *analyzer.PropUpdate {
return nil
}
// Parse OpenVPN UDP packet.
func (o *openvpnUDPStream) parsePkt() (p *openvpnPkt, action utils.LSMAction) {
if o.curPkt == nil {
return nil, utils.LSMActionPause
}
if !OpenVPNCheckForValidOpcode(o.curPkt[0] >> 3) {
return nil, utils.LSMActionCancel
}
// Parse packet header
p = &openvpnPkt{}
p.opcode = o.curPkt[0] >> 3
p._keyId = o.curPkt[0] & 0x07
o.curPkt = nil
return p, utils.LSMActionNext
}
type openvpnTCPStream struct {
openvpnStream
reqBuf *utils.ByteBuffer
respBuf *utils.ByteBuffer
}
func newOpenVPNTCPStream(logger analyzer.Logger) *openvpnTCPStream {
s := &openvpnTCPStream{
openvpnStream: openvpnStream{
logger: logger,
pktLimit: OpenVPNTCPPktDefaultLimit,
},
reqBuf: &utils.ByteBuffer{},
respBuf: &utils.ByteBuffer{},
}
s.respPktParse = func() (*openvpnPkt, utils.LSMAction) {
return s.parsePkt(true)
}
s.reqPktParse = func() (*openvpnPkt, utils.LSMAction) {
return s.parsePkt(false)
}
s.reqLSM = utils.NewLinearStateMachine(
s.parseCtlHardResetClient,
s.parseReq,
)
s.respLSM = utils.NewLinearStateMachine(
s.parseCtlHardResetServer,
s.parseResp,
)
return s
}
func (o *openvpnTCPStream) Feed(rev, start, end bool, skip int, data []byte) (u *analyzer.PropUpdate, d bool) {
if skip != 0 {
return nil, true
}
if len(data) == 0 {
return nil, false
}
var update *analyzer.PropUpdate
var cancelled bool
if rev {
o.respBuf.Append(data)
o.respUpdated = false
cancelled, o.respDone = o.respLSM.Run()
if o.respUpdated {
update = &analyzer.PropUpdate{
Type: analyzer.PropUpdateReplace,
M: analyzer.PropMap{"rx_pkt_cnt": o.rxPktCnt, "tx_pkt_cnt": o.txPktCnt},
}
o.respUpdated = false
}
} else {
o.reqBuf.Append(data)
o.reqUpdated = false
cancelled, o.reqDone = o.reqLSM.Run()
if o.reqUpdated {
update = &analyzer.PropUpdate{
Type: analyzer.PropUpdateMerge,
M: analyzer.PropMap{"rx_pkt_cnt": o.rxPktCnt, "tx_pkt_cnt": o.txPktCnt},
}
o.reqUpdated = false
}
}
return update, cancelled || (o.reqDone && o.respDone) || o.rxPktCnt+o.txPktCnt > o.pktLimit
}
func (o *openvpnTCPStream) Close(limited bool) *analyzer.PropUpdate {
o.reqBuf.Reset()
o.respBuf.Reset()
return nil
}
// Parse OpenVPN TCP packet.
func (o *openvpnTCPStream) parsePkt(rev bool) (p *openvpnPkt, action utils.LSMAction) {
var buffer *utils.ByteBuffer
if rev {
buffer = o.respBuf
} else {
buffer = o.reqBuf
}
// Parse packet length
pktLen, ok := buffer.GetUint16(false, false)
if !ok {
return nil, utils.LSMActionPause
}
if pktLen < OpenVPNMinPktLen {
return nil, utils.LSMActionCancel
}
pktOp, ok := buffer.Get(3, false)
if !ok {
return nil, utils.LSMActionPause
}
if !OpenVPNCheckForValidOpcode(pktOp[2] >> 3) {
return nil, utils.LSMActionCancel
}
pkt, ok := buffer.Get(int(pktLen)+2, true)
if !ok {
return nil, utils.LSMActionPause
}
pkt = pkt[2:]
// Parse packet header
p = &openvpnPkt{}
p.pktLen = pktLen
p.opcode = pkt[0] >> 3
p._keyId = pkt[0] & 0x07
return p, utils.LSMActionNext
}
func OpenVPNCheckForValidOpcode(opcode byte) bool {
switch opcode {
case OpenVPNControlHardResetClientV1,
OpenVPNControlHardResetServerV1,
OpenVPNControlSoftResetV1,
OpenVPNControlV1,
OpenVPNAckV1,
OpenVPNDataV1,
OpenVPNControlHardResetClientV2,
OpenVPNControlHardResetServerV2,
OpenVPNDataV2,
OpenVPNControlHardResetClientV3,
OpenVPNControlWkcV1:
return true
}
return false
}

View File

@@ -36,41 +36,40 @@ type quicStream struct {
}
func (s *quicStream) Feed(rev bool, data []byte) (u *analyzer.PropUpdate, done bool) {
// minimal data size: protocol version (2 bytes) + random (32 bytes) +
// + session ID (1 byte) + cipher suites (4 bytes) +
// + compression methods (2 bytes) + no extensions
const minDataSize = 41
if rev {
// We don't support server direction for now
s.invalidCount++
return nil, s.invalidCount >= quicInvalidCountThreshold
}
pl, err := quic.ReadCryptoPayload(data)
if err != nil || len(pl) < 4 {
if err != nil || len(pl) < 4 { // FIXME: isn't length checked inside quic.ReadCryptoPayload? Also, what about error handling?
s.invalidCount++
return nil, s.invalidCount >= quicInvalidCountThreshold
}
// Should be a TLS client hello
if pl[0] != 0x01 {
// Not a client hello
if pl[0] != internal.TypeClientHello {
s.invalidCount++
return nil, s.invalidCount >= quicInvalidCountThreshold
}
chLen := int(pl[1])<<16 | int(pl[2])<<8 | int(pl[3])
if chLen < 41 {
// 2 (Protocol Version) +
// 32 (Random) +
// 1 (Session ID Length) +
// 2 (Cipher Suites Length) +_ws.col.protocol == "TLSv1.3"
// 2 (Cipher Suite) +
// 1 (Compression Methods Length) +
// 1 (Compression Method) +
// No extensions
// This should be the bare minimum for a client hello
if chLen < minDataSize {
s.invalidCount++
return nil, s.invalidCount >= quicInvalidCountThreshold
}
m := internal.ParseTLSClientHello(&utils.ByteBuffer{Buf: pl[4:]})
m := internal.ParseTLSClientHelloMsgData(&utils.ByteBuffer{Buf: pl[4:]})
if m == nil {
s.invalidCount++
return nil, s.invalidCount >= quicInvalidCountThreshold
}
return &analyzer.PropUpdate{
Type: analyzer.PropUpdateMerge,
M: analyzer.PropMap{"req": m},

58
analyzer/udp/quic_test.go Normal file
View File

@@ -0,0 +1,58 @@
package udp
import (
"reflect"
"testing"
"github.com/apernet/OpenGFW/analyzer"
)
func TestQuicStreamParsing_ClientHello(t *testing.T) {
// example packet taken from <https://quic.xargs.org/#client-initial-packet/annotated>
clientHello := make([]byte, 1200)
clientInitial := []byte{
0xcd, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
0x06, 0x07, 0x05, 0x63, 0x5f, 0x63, 0x69, 0x64, 0x00, 0x41, 0x03, 0x98,
0x1c, 0x36, 0xa7, 0xed, 0x78, 0x71, 0x6b, 0xe9, 0x71, 0x1b, 0xa4, 0x98,
0xb7, 0xed, 0x86, 0x84, 0x43, 0xbb, 0x2e, 0x0c, 0x51, 0x4d, 0x4d, 0x84,
0x8e, 0xad, 0xcc, 0x7a, 0x00, 0xd2, 0x5c, 0xe9, 0xf9, 0xaf, 0xa4, 0x83,
0x97, 0x80, 0x88, 0xde, 0x83, 0x6b, 0xe6, 0x8c, 0x0b, 0x32, 0xa2, 0x45,
0x95, 0xd7, 0x81, 0x3e, 0xa5, 0x41, 0x4a, 0x91, 0x99, 0x32, 0x9a, 0x6d,
0x9f, 0x7f, 0x76, 0x0d, 0xd8, 0xbb, 0x24, 0x9b, 0xf3, 0xf5, 0x3d, 0x9a,
0x77, 0xfb, 0xb7, 0xb3, 0x95, 0xb8, 0xd6, 0x6d, 0x78, 0x79, 0xa5, 0x1f,
0xe5, 0x9e, 0xf9, 0x60, 0x1f, 0x79, 0x99, 0x8e, 0xb3, 0x56, 0x8e, 0x1f,
0xdc, 0x78, 0x9f, 0x64, 0x0a, 0xca, 0xb3, 0x85, 0x8a, 0x82, 0xef, 0x29,
0x30, 0xfa, 0x5c, 0xe1, 0x4b, 0x5b, 0x9e, 0xa0, 0xbd, 0xb2, 0x9f, 0x45,
0x72, 0xda, 0x85, 0xaa, 0x3d, 0xef, 0x39, 0xb7, 0xef, 0xaf, 0xff, 0xa0,
0x74, 0xb9, 0x26, 0x70, 0x70, 0xd5, 0x0b, 0x5d, 0x07, 0x84, 0x2e, 0x49,
0xbb, 0xa3, 0xbc, 0x78, 0x7f, 0xf2, 0x95, 0xd6, 0xae, 0x3b, 0x51, 0x43,
0x05, 0xf1, 0x02, 0xaf, 0xe5, 0xa0, 0x47, 0xb3, 0xfb, 0x4c, 0x99, 0xeb,
0x92, 0xa2, 0x74, 0xd2, 0x44, 0xd6, 0x04, 0x92, 0xc0, 0xe2, 0xe6, 0xe2,
0x12, 0xce, 0xf0, 0xf9, 0xe3, 0xf6, 0x2e, 0xfd, 0x09, 0x55, 0xe7, 0x1c,
0x76, 0x8a, 0xa6, 0xbb, 0x3c, 0xd8, 0x0b, 0xbb, 0x37, 0x55, 0xc8, 0xb7,
0xeb, 0xee, 0x32, 0x71, 0x2f, 0x40, 0xf2, 0x24, 0x51, 0x19, 0x48, 0x70,
0x21, 0xb4, 0xb8, 0x4e, 0x15, 0x65, 0xe3, 0xca, 0x31, 0x96, 0x7a, 0xc8,
0x60, 0x4d, 0x40, 0x32, 0x17, 0x0d, 0xec, 0x28, 0x0a, 0xee, 0xfa, 0x09,
0x5d, 0x08, 0xb3, 0xb7, 0x24, 0x1e, 0xf6, 0x64, 0x6a, 0x6c, 0x86, 0xe5,
0xc6, 0x2c, 0xe0, 0x8b, 0xe0, 0x99,
}
copy(clientHello, clientInitial)
want := analyzer.PropMap{
"alpn": []string{"ping/1.0"},
"ciphers": []uint16{4865, 4866, 4867},
"compression": []uint8{0},
"random": []uint8{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31},
"session": []uint8{},
"sni": "example.ulfheim.net",
"supported_versions": []uint16{772},
"version": uint16(771),
}
s := quicStream{}
u, _ := s.Feed(false, clientHello)
got := u.M.Get("req")
if !reflect.DeepEqual(got, want) {
t.Errorf("%d B parsed = %v, want %v", len(clientHello), got, want)
}
}

View File

@@ -5,9 +5,9 @@ import (
"fmt"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/apernet/OpenGFW/analyzer"
"github.com/apernet/OpenGFW/analyzer/tcp"
@@ -17,6 +17,7 @@ import (
"github.com/apernet/OpenGFW/modifier"
modUDP "github.com/apernet/OpenGFW/modifier/udp"
"github.com/apernet/OpenGFW/ruleset"
"github.com/apernet/OpenGFW/ruleset/builtins/geo"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -42,6 +43,7 @@ var logger *zap.Logger
// Flags
var (
cfgFile string
pcapFile string
logLevel string
logFormat string
)
@@ -93,6 +95,7 @@ var analyzers = []analyzer.Analyzer{
&tcp.TLSAnalyzer{},
&tcp.TrojanAnalyzer{},
&udp.DNSAnalyzer{},
&udp.OpenVPNAnalyzer{},
&udp.QUICAnalyzer{},
&udp.WireGuardAnalyzer{},
}
@@ -116,6 +119,7 @@ func init() {
func initFlags() {
rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file")
rootCmd.PersistentFlags().StringVarP(&pcapFile, "pcap", "p", "", "pcap file (optional)")
rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", envOrDefaultString(appLogLevelEnv, "info"), "log level")
rootCmd.PersistentFlags().StringVarP(&logFormat, "log-format", "f", envOrDefaultString(appLogFormatEnv, "console"), "log format")
}
@@ -165,19 +169,28 @@ type cliConfig struct {
IO cliConfigIO `mapstructure:"io"`
Workers cliConfigWorkers `mapstructure:"workers"`
Ruleset cliConfigRuleset `mapstructure:"ruleset"`
Replay cliConfigReplay `mapstructure:"replay"`
}
type cliConfigIO struct {
QueueSize uint32 `mapstructure:"queueSize"`
Local bool `mapstructure:"local"`
QueueSize uint32 `mapstructure:"queueSize"`
ReadBuffer int `mapstructure:"rcvBuf"`
WriteBuffer int `mapstructure:"sndBuf"`
Local bool `mapstructure:"local"`
RST bool `mapstructure:"rst"`
}
type cliConfigReplay struct {
Realtime bool `mapstructure:"realtime"`
}
type cliConfigWorkers struct {
Count int `mapstructure:"count"`
QueueSize int `mapstructure:"queueSize"`
TCPMaxBufferedPagesTotal int `mapstructure:"tcpMaxBufferedPagesTotal"`
TCPMaxBufferedPagesPerConn int `mapstructure:"tcpMaxBufferedPagesPerConn"`
UDPMaxStreams int `mapstructure:"udpMaxStreams"`
Count int `mapstructure:"count"`
QueueSize int `mapstructure:"queueSize"`
TCPMaxBufferedPagesTotal int `mapstructure:"tcpMaxBufferedPagesTotal"`
TCPMaxBufferedPagesPerConn int `mapstructure:"tcpMaxBufferedPagesPerConn"`
TCPTimeout time.Duration `mapstructure:"tcpTimeout"`
UDPMaxStreams int `mapstructure:"udpMaxStreams"`
}
type cliConfigRuleset struct {
@@ -191,14 +204,30 @@ func (c *cliConfig) fillLogger(config *engine.Config) error {
}
func (c *cliConfig) fillIO(config *engine.Config) error {
nfio, err := io.NewNFQueuePacketIO(io.NFQueuePacketIOConfig{
QueueSize: c.IO.QueueSize,
Local: c.IO.Local,
})
var ioImpl io.PacketIO
var err error
if pcapFile != "" {
// Setup IO for pcap file replay
logger.Info("replaying from pcap file", zap.String("pcap file", pcapFile))
ioImpl, err = io.NewPcapPacketIO(io.PcapPacketIOConfig{
PcapFile: pcapFile,
Realtime: c.Replay.Realtime,
})
} else {
// Setup IO for nfqueue
ioImpl, err = io.NewNFQueuePacketIO(io.NFQueuePacketIOConfig{
QueueSize: c.IO.QueueSize,
ReadBuffer: c.IO.ReadBuffer,
WriteBuffer: c.IO.WriteBuffer,
Local: c.IO.Local,
RST: c.IO.RST,
})
}
if err != nil {
return configError{Field: "io", Err: err}
}
config.IOs = []io.PacketIO{nfio}
config.IO = ioImpl
return nil
}
@@ -207,6 +236,7 @@ func (c *cliConfig) fillWorkers(config *engine.Config) error {
config.WorkerQueueSize = c.Workers.QueueSize
config.WorkerTCPMaxBufferedPagesTotal = c.Workers.TCPMaxBufferedPagesTotal
config.WorkerTCPMaxBufferedPagesPerConn = c.Workers.TCPMaxBufferedPagesPerConn
config.WorkerTCPTimeout = c.Workers.TCPTimeout
config.WorkerUDPMaxStreams = c.Workers.UDPMaxStreams
return nil
}
@@ -241,12 +271,7 @@ func runMain(cmd *cobra.Command, args []string) {
if err != nil {
logger.Fatal("failed to parse config", zap.Error(err))
}
defer func() {
// Make sure to close all IOs on exit
for _, i := range engineConfig.IOs {
_ = i.Close()
}
}()
defer engineConfig.IO.Close() // Make sure to close IO on exit
// Ruleset
rawRs, err := ruleset.ExprRulesFromYAML(args[0])
@@ -254,9 +279,9 @@ func runMain(cmd *cobra.Command, args []string) {
logger.Fatal("failed to load rules", zap.Error(err))
}
rsConfig := &ruleset.BuiltinConfig{
Logger: &rulesetLogger{},
GeoSiteFilename: config.Ruleset.GeoSite,
GeoIpFilename: config.Ruleset.GeoIp,
Logger: &rulesetLogger{},
GeoMatcher: geo.NewGeoMatcher(config.Ruleset.GeoSite, config.Ruleset.GeoIp),
ProtectedDialContext: engineConfig.IO.ProtectedDialContext,
}
rs, err := ruleset.CompileExprRules(rawRs, analyzers, modifiers, rsConfig)
if err != nil {
@@ -274,15 +299,15 @@ func runMain(cmd *cobra.Command, args []string) {
ctx, cancelFunc := context.WithCancel(context.Background())
go func() {
// Graceful shutdown
shutdownChan := make(chan os.Signal)
signal.Notify(shutdownChan, os.Interrupt, os.Kill)
shutdownChan := make(chan os.Signal, 1)
signal.Notify(shutdownChan, os.Interrupt, syscall.SIGTERM)
<-shutdownChan
logger.Info("shutting down gracefully...")
cancelFunc()
}()
go func() {
// Rule reload
reloadChan := make(chan os.Signal)
reloadChan := make(chan os.Signal, 1)
signal.Notify(reloadChan, syscall.SIGHUP)
for {
<-reloadChan
@@ -338,12 +363,26 @@ func (l *engineLogger) TCPStreamPropUpdate(info ruleset.StreamInfo, close bool)
}
func (l *engineLogger) TCPStreamAction(info ruleset.StreamInfo, action ruleset.Action, noMatch bool) {
logger.Info("TCP stream action",
zap.Int64("id", info.ID),
zap.String("src", info.SrcString()),
zap.String("dst", info.DstString()),
zap.String("action", action.String()),
zap.Bool("noMatch", noMatch))
if noMatch {
logger.Debug("TCP stream no match",
zap.Int64("id", info.ID),
zap.String("src", info.SrcString()),
zap.String("dst", info.DstString()),
zap.String("action", action.String()))
} else {
logger.Info("TCP stream action",
zap.Int64("id", info.ID),
zap.String("src", info.SrcString()),
zap.String("dst", info.DstString()),
zap.String("action", action.String()))
}
}
func (l *engineLogger) TCPFlush(workerID, flushed, closed int) {
logger.Debug("TCP flush",
zap.Int("workerID", workerID),
zap.Int("flushed", flushed),
zap.Int("closed", closed))
}
func (l *engineLogger) UDPStreamNew(workerID int, info ruleset.StreamInfo) {
@@ -364,12 +403,19 @@ func (l *engineLogger) UDPStreamPropUpdate(info ruleset.StreamInfo, close bool)
}
func (l *engineLogger) UDPStreamAction(info ruleset.StreamInfo, action ruleset.Action, noMatch bool) {
logger.Info("UDP stream action",
zap.Int64("id", info.ID),
zap.String("src", info.SrcString()),
zap.String("dst", info.DstString()),
zap.String("action", action.String()),
zap.Bool("noMatch", noMatch))
if noMatch {
logger.Debug("UDP stream no match",
zap.Int64("id", info.ID),
zap.String("src", info.SrcString()),
zap.String("dst", info.DstString()),
zap.String("action", action.String()))
} else {
logger.Info("UDP stream action",
zap.Int64("id", info.ID),
zap.String("src", info.SrcString()),
zap.String("dst", info.DstString()),
zap.String("action", action.String()))
}
}
func (l *engineLogger) ModifyError(info ruleset.StreamInfo, err error) {
@@ -427,11 +473,3 @@ func envOrDefaultString(key, def string) string {
}
return def
}
func envOrDefaultBool(key string, def bool) bool {
if v := os.Getenv(key); v != "" {
b, _ := strconv.ParseBool(v)
return b
}
return def
}

View File

@@ -1,418 +0,0 @@
# Analyzers
Analyzers are one of the main components of OpenGFW. Their job is to analyze a connection, see if it's a protocol they
support, and if so, extract information from that connection and provide properties for the rule engine to match against
user-provided rules. OpenGFW will automatically analyze which analyzers are referenced in the given rules and enable
only those that are needed.
This document lists the properties provided by each analyzer that can be used by rules.
## DNS (TCP & UDP)
For queries:
```json
{
"dns": {
"aa": false,
"id": 41953,
"opcode": 0,
"qr": false,
"questions": [
{
"class": 1,
"name": "www.google.com",
"type": 1
}
],
"ra": false,
"rcode": 0,
"rd": true,
"tc": false,
"z": 0
}
}
```
For responses:
```json
{
"dns": {
"aa": false,
"answers": [
{
"a": "142.251.32.36",
"class": 1,
"name": "www.google.com",
"ttl": 255,
"type": 1
}
],
"id": 41953,
"opcode": 0,
"qr": true,
"questions": [
{
"class": 1,
"name": "www.google.com",
"type": 1
}
],
"ra": true,
"rcode": 0,
"rd": true,
"tc": false,
"z": 0
}
}
```
Example for blocking DNS queries for `www.google.com`:
```yaml
- name: Block Google DNS
action: drop
expr: dns != nil && !dns.qr && any(dns.questions, {.name == "www.google.com"})
```
## FET (Fully Encrypted Traffic)
Check https://www.usenix.org/system/files/usenixsecurity23-wu-mingshi.pdf for more information.
```json
{
"fet": {
"ex1": 3.7560976,
"ex2": true,
"ex3": 0.9512195,
"ex4": 39,
"ex5": false,
"yes": false
}
}
```
Example for blocking fully encrypted traffic:
```yaml
- name: Block suspicious proxy traffic
action: block
expr: fet != nil && fet.yes
```
## HTTP
```json
{
"http": {
"req": {
"headers": {
"accept": "*/*",
"host": "ipinfo.io",
"user-agent": "curl/7.81.0"
},
"method": "GET",
"path": "/",
"version": "HTTP/1.1"
},
"resp": {
"headers": {
"access-control-allow-origin": "*",
"content-length": "333",
"content-type": "application/json; charset=utf-8",
"date": "Wed, 24 Jan 2024 05:41:44 GMT",
"referrer-policy": "strict-origin-when-cross-origin",
"server": "nginx/1.24.0",
"strict-transport-security": "max-age=2592000; includeSubDomains",
"via": "1.1 google",
"x-content-type-options": "nosniff",
"x-envoy-upstream-service-time": "2",
"x-frame-options": "SAMEORIGIN",
"x-xss-protection": "1; mode=block"
},
"status": 200,
"version": "HTTP/1.1"
}
}
}
```
Example for blocking HTTP requests to `ipinfo.io`:
```yaml
- name: Block ipinfo.io HTTP
action: block
expr: http != nil && http.req != nil && http.req.headers != nil && http.req.headers.host == "ipinfo.io"
```
## SSH
```json
{
"ssh": {
"server": {
"comments": "Ubuntu-3ubuntu0.6",
"protocol": "2.0",
"software": "OpenSSH_8.9p1"
},
"client": {
"comments": "IMHACKER",
"protocol": "2.0",
"software": "OpenSSH_8.9p1"
}
}
}
```
Example for blocking all SSH connections:
```yaml
- name: Block SSH
action: block
expr: ssh != nil
```
## TLS
```json
{
"tls": {
"req": {
"alpn": ["h2", "http/1.1"],
"ciphers": [
4866, 4867, 4865, 49196, 49200, 159, 52393, 52392, 52394, 49195, 49199,
158, 49188, 49192, 107, 49187, 49191, 103, 49162, 49172, 57, 49161,
49171, 51, 157, 156, 61, 60, 53, 47, 255
],
"compression": "AA==",
"random": "UqfPi+EmtMgusILrKcELvVWwpOdPSM/My09nPXl84dg=",
"session": "jCTrpAzHpwrfuYdYx4FEjZwbcQxCuZ52HGIoOcbw1vA=",
"sni": "ipinfo.io",
"supported_versions": [772, 771],
"version": 771,
"ech": true
},
"resp": {
"cipher": 4866,
"compression": 0,
"random": "R/Cy1m9pktuBMZQIHahD8Y83UWPRf8j8luwNQep9yJI=",
"session": "jCTrpAzHpwrfuYdYx4FEjZwbcQxCuZ52HGIoOcbw1vA=",
"supported_versions": 772,
"version": 771
}
}
}
```
Example for blocking TLS connections to `ipinfo.io`:
```yaml
- name: Block ipinfo.io TLS
action: block
expr: tls != nil && tls.req != nil && tls.req.sni == "ipinfo.io"
```
## QUIC
QUIC analyzer produces the same result format as TLS analyzer, but currently only supports "req" direction (client
hello), not "resp" (server hello).
```json
{
"quic": {
"req": {
"alpn": ["h3"],
"ciphers": [4865, 4866, 4867],
"compression": "AA==",
"ech": true,
"random": "FUYLceFReLJl9dRQ0HAus7fi2ZGuKIAApF4keeUqg00=",
"session": "",
"sni": "quic.rocks",
"supported_versions": [772],
"version": 771
}
}
}
```
Example for blocking QUIC connections to `quic.rocks`:
```yaml
- name: Block quic.rocks QUIC
action: block
expr: quic != nil && quic.req != nil && quic.req.sni == "quic.rocks"
```
## Trojan (proxy protocol)
Check https://github.com/XTLS/Trojan-killer for more information.
```json
{
"trojan": {
"down": 4712,
"up": 671,
"yes": true
}
}
```
Example for blocking Trojan connections:
```yaml
- name: Block Trojan
action: block
expr: trojan != nil && trojan.yes
```
## SOCKS
SOCKS4:
```json
{
"socks": {
"version": 4,
"req": {
"cmd": 1,
"addr_type": 1, // same as socks5
"addr": "1.1.1.1",
// for socks4a
// "addr_type": 3,
// "addr": "google.com",
"port": 443,
"auth": {
"user_id": "user"
}
},
"resp": {
"rep": 90, // 0x5A(90) granted
"addr_type": 1,
"addr": "1.1.1.1",
"port": 443
}
}
}
```
SOCKS5 without auth:
```json
{
"socks": {
"version": 5,
"req": {
"cmd": 1, // 0x01: connect, 0x02: bind, 0x03: udp
"addr_type": 3, // 0x01: ipv4, 0x03: domain, 0x04: ipv6
"addr": "google.com",
"port": 80,
"auth": {
"method": 0 // 0x00: no auth, 0x02: username/password
}
},
"resp": {
"rep": 0, // 0x00: success
"addr_type": 1, // 0x01: ipv4, 0x03: domain, 0x04: ipv6
"addr": "198.18.1.31",
"port": 80,
"auth": {
"method": 0 // 0x00: no auth, 0x02: username/password
}
}
}
}
```
SOCKS5 with auth:
```json
{
"socks": {
"version": 5,
"req": {
"cmd": 1, // 0x01: connect, 0x02: bind, 0x03: udp
"addr_type": 3, // 0x01: ipv4, 0x03: domain, 0x04: ipv6
"addr": "google.com",
"port": 80,
"auth": {
"method": 2, // 0x00: no auth, 0x02: username/password
"username": "user",
"password": "pass"
}
},
"resp": {
"rep": 0, // 0x00: success
"addr_type": 1, // 0x01: ipv4, 0x03: domain, 0x04: ipv6
"addr": "198.18.1.31",
"port": 80,
"auth": {
"method": 2, // 0x00: no auth, 0x02: username/password
"status": 0 // 0x00: success, 0x01: failure
}
}
}
}
```
Example for blocking connections to `google.com:80` and user `foobar`:
```yaml
- name: Block SOCKS google.com:80
action: block
expr: string(socks?.req?.addr) endsWith "google.com" && socks?.req?.port == 80
- name: Block SOCKS user foobar
action: block
expr: socks?.req?.auth?.method == 2 && socks?.req?.auth?.username == "foobar"
```
## WireGuard
```json
{
"wireguard": {
"message_type": 1, // 0x1: handshake_initiation, 0x2: handshake_response, 0x3: packet_cookie_reply, 0x4: packet_data
"handshake_initiation": {
"sender_index": 0x12345678
},
"handshake_response": {
"sender_index": 0x12345678,
"receiver_index": 0x87654321,
"receiver_index_matched": true
},
"packet_data": {
"receiver_index": 0x12345678,
"receiver_index_matched": true
},
"packet_cookie_reply": {
"receiver_index": 0x12345678,
"receiver_index_matched": true
}
}
}
```
Example for blocking WireGuard traffic:
```yaml
# false positive: high
- name: Block all WireGuard-like traffic
action: block
expr: wireguard != nil
# false positive: medium
- name: Block WireGuard by handshake_initiation
action: drop
expr: wireguard?.handshake_initiation != nil
# false positive: low
- name: Block WireGuard by handshake_response
action: drop
expr: wireguard?.handshake_response?.receiver_index_matched == true
# false positive: pretty low
- name: Block WireGuard by packet_data
action: block
expr: wireguard?.packet_data?.receiver_index_matched == true
```

View File

@@ -15,7 +15,7 @@ var _ Engine = (*engine)(nil)
type engine struct {
logger Logger
ioList []io.PacketIO
io io.PacketIO
workers []*worker
}
@@ -34,6 +34,7 @@ func NewEngine(config Config) (Engine, error) {
Ruleset: config.Ruleset,
TCPMaxBufferedPagesTotal: config.WorkerTCPMaxBufferedPagesTotal,
TCPMaxBufferedPagesPerConn: config.WorkerTCPMaxBufferedPagesPerConn,
TCPTimeout: config.WorkerTCPTimeout,
UDPMaxStreams: config.WorkerUDPMaxStreams,
})
if err != nil {
@@ -42,7 +43,7 @@ func NewEngine(config Config) (Engine, error) {
}
return &engine{
logger: config.Logger,
ioList: config.IOs,
io: config.IO,
workers: workers,
}, nil
}
@@ -57,28 +58,30 @@ func (e *engine) UpdateRuleset(r ruleset.Ruleset) error {
}
func (e *engine) Run(ctx context.Context) error {
workerCtx, workerCancel := context.WithCancel(ctx)
defer workerCancel() // Stop workers
// Register IO shutdown
ioCtx, ioCancel := context.WithCancel(ctx)
defer ioCancel() // Stop workers & IOs
e.io.SetCancelFunc(ioCancel)
defer ioCancel() // Stop IO
// Start workers
for _, w := range e.workers {
go w.Run(ioCtx)
go w.Run(workerCtx)
}
// Register callbacks
errChan := make(chan error, len(e.ioList))
for _, i := range e.ioList {
ioEntry := i // Make sure dispatch() uses the correct ioEntry
err := ioEntry.Register(ioCtx, func(p io.Packet, err error) bool {
if err != nil {
errChan <- err
return false
}
return e.dispatch(ioEntry, p)
})
// Register IO callback
errChan := make(chan error, 1)
err := e.io.Register(ioCtx, func(p io.Packet, err error) bool {
if err != nil {
return err
errChan <- err
return false
}
return e.dispatch(p)
})
if err != nil {
return err
}
// Block until IO errors or context is cancelled
@@ -87,12 +90,13 @@ func (e *engine) Run(ctx context.Context) error {
return err
case <-ctx.Done():
return nil
case <-ioCtx.Done():
return nil
}
}
// dispatch dispatches a packet to a worker.
// This must be safe for concurrent use, as it may be called from multiple IOs.
func (e *engine) dispatch(ioEntry io.PacketIO, p io.Packet) bool {
func (e *engine) dispatch(p io.Packet) bool {
data := p.Data()
ipVersion := data[0] >> 4
var layerType gopacket.LayerType
@@ -102,17 +106,19 @@ func (e *engine) dispatch(ioEntry io.PacketIO, p io.Packet) bool {
layerType = layers.LayerTypeIPv6
} else {
// Unsupported network layer
_ = ioEntry.SetVerdict(p, io.VerdictAcceptStream, nil)
_ = e.io.SetVerdict(p, io.VerdictAcceptStream, nil)
return true
}
// Convert to gopacket.Packet
packet := gopacket.NewPacket(data, layerType, gopacket.DecodeOptions{Lazy: true, NoCopy: true})
packet.Metadata().Timestamp = p.Timestamp()
// Load balance by stream ID
index := p.StreamID() % uint32(len(e.workers))
packet := gopacket.NewPacket(data, layerType, gopacket.DecodeOptions{Lazy: true, NoCopy: true})
e.workers[index].Feed(&workerPacket{
StreamID: p.StreamID(),
Packet: packet,
SetVerdict: func(v io.Verdict, b []byte) error {
return ioEntry.SetVerdict(p, v, b)
return e.io.SetVerdict(p, v, b)
},
})
return true

View File

@@ -2,6 +2,7 @@ package engine
import (
"context"
"time"
"github.com/apernet/OpenGFW/io"
"github.com/apernet/OpenGFW/ruleset"
@@ -18,13 +19,14 @@ type Engine interface {
// Config is the configuration for the engine.
type Config struct {
Logger Logger
IOs []io.PacketIO
IO io.PacketIO
Ruleset ruleset.Ruleset
Workers int // Number of workers. Zero or negative means auto (number of CPU cores).
WorkerQueueSize int
WorkerTCPMaxBufferedPagesTotal int
WorkerTCPMaxBufferedPagesPerConn int
WorkerTCPTimeout time.Duration
WorkerUDPMaxStreams int
}
@@ -36,6 +38,7 @@ type Logger interface {
TCPStreamNew(workerID int, info ruleset.StreamInfo)
TCPStreamPropUpdate(info ruleset.StreamInfo, close bool)
TCPStreamAction(info ruleset.StreamInfo, action ruleset.Action, noMatch bool)
TCPFlush(workerID, flushed, closed int)
UDPStreamNew(workerID int, info ruleset.StreamInfo)
UDPStreamPropUpdate(info ruleset.StreamInfo, close bool)

View File

@@ -2,6 +2,7 @@ package engine
import (
"context"
"time"
"github.com/apernet/OpenGFW/io"
"github.com/apernet/OpenGFW/ruleset"
@@ -14,9 +15,12 @@ import (
const (
defaultChanSize = 64
defaultTCPMaxBufferedPagesTotal = 4096
defaultTCPMaxBufferedPagesPerConnection = 64
defaultTCPMaxBufferedPagesTotal = 65536
defaultTCPMaxBufferedPagesPerConnection = 16
defaultTCPTimeout = 10 * time.Minute
defaultUDPMaxStreams = 4096
tcpFlushInterval = 1 * time.Minute
)
type workerPacket struct {
@@ -33,6 +37,7 @@ type worker struct {
tcpStreamFactory *tcpStreamFactory
tcpStreamPool *reassembly.StreamPool
tcpAssembler *reassembly.Assembler
tcpTimeout time.Duration
udpStreamFactory *udpStreamFactory
udpStreamManager *udpStreamManager
@@ -47,6 +52,7 @@ type workerConfig struct {
Ruleset ruleset.Ruleset
TCPMaxBufferedPagesTotal int
TCPMaxBufferedPagesPerConn int
TCPTimeout time.Duration
UDPMaxStreams int
}
@@ -60,6 +66,9 @@ func (c *workerConfig) fillDefaults() {
if c.TCPMaxBufferedPagesPerConn <= 0 {
c.TCPMaxBufferedPagesPerConn = defaultTCPMaxBufferedPagesPerConnection
}
if c.TCPTimeout <= 0 {
c.TCPTimeout = defaultTCPTimeout
}
if c.UDPMaxStreams <= 0 {
c.UDPMaxStreams = defaultUDPMaxStreams
}
@@ -98,6 +107,7 @@ func newWorker(config workerConfig) (*worker, error) {
tcpStreamFactory: tcpSF,
tcpStreamPool: tcpStreamPool,
tcpAssembler: tcpAssembler,
tcpTimeout: config.TCPTimeout,
udpStreamFactory: udpSF,
udpStreamManager: udpSM,
modSerializeBuffer: gopacket.NewSerializeBuffer(),
@@ -111,6 +121,10 @@ func (w *worker) Feed(p *workerPacket) {
func (w *worker) Run(ctx context.Context) {
w.logger.WorkerStart(w.id)
defer w.logger.WorkerStop(w.id)
tcpFlushTicker := time.NewTicker(tcpFlushInterval)
defer tcpFlushTicker.Stop()
for {
select {
case <-ctx.Done():
@@ -122,6 +136,8 @@ func (w *worker) Run(ctx context.Context) {
}
v, b := w.handle(wPkt.StreamID, wPkt.Packet)
_ = wPkt.SetVerdict(v, b)
case <-tcpFlushTicker.C:
w.flushTCP(w.tcpTimeout)
}
}
}
@@ -176,6 +192,11 @@ func (w *worker) handleTCP(ipFlow gopacket.Flow, pMeta *gopacket.PacketMetadata,
return io.Verdict(ctx.Verdict)
}
func (w *worker) flushTCP(timeout time.Duration) {
flushed, closed := w.tcpAssembler.FlushCloseOlderThan(time.Now().Add(-timeout))
w.logger.TCPFlush(w.id, flushed, closed)
}
func (w *worker) handleUDP(streamID uint32, ipFlow gopacket.Flow, udp *layers.UDP) (io.Verdict, []byte) {
ctx := &udpContext{
Verdict: udpVerdictAccept,

5
go.mod
View File

@@ -5,7 +5,7 @@ go 1.21
require (
github.com/bwmarrin/snowflake v0.3.0
github.com/coreos/go-iptables v0.7.0
github.com/expr-lang/expr v1.15.7
github.com/expr-lang/expr v1.16.3
github.com/florianl/go-nfqueue v1.3.2-0.20231218173729-f2bdeb033acf
github.com/google/gopacket v1.1.20-0.20220810144506-32ee38206866
github.com/hashicorp/golang-lru/v2 v2.0.7
@@ -13,7 +13,6 @@ require (
github.com/quic-go/quic-go v0.41.0
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
github.com/stretchr/testify v1.8.4
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.19.0
golang.org/x/sys v0.17.0
@@ -22,7 +21,6 @@ require (
)
require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
@@ -32,7 +30,6 @@ require (
github.com/mdlayher/socket v0.1.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect

4
go.sum
View File

@@ -7,8 +7,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/expr-lang/expr v1.15.7 h1:BK0JcWUkoW6nrbLBo6xCKhz4BvH5DSOOu1Gx5lucyZo=
github.com/expr-lang/expr v1.15.7/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ=
github.com/expr-lang/expr v1.16.3 h1:NLldf786GffptcXNxxJx5dQ+FzeWDKChBDqOOwyK8to=
github.com/expr-lang/expr v1.16.3/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ=
github.com/florianl/go-nfqueue v1.3.2-0.20231218173729-f2bdeb033acf h1:NqGS3vTHzVENbIfd87cXZwdpO6MB2R1PjHMJLi4Z3ow=
github.com/florianl/go-nfqueue v1.3.2-0.20231218173729-f2bdeb033acf/go.mod h1:eSnAor2YCfMCVYrVNEhkLGN/r1L+J4uDjc0EUy0tfq4=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=

View File

@@ -2,6 +2,8 @@ package io
import (
"context"
"net"
"time"
)
type Verdict int
@@ -23,13 +25,14 @@ const (
type Packet interface {
// StreamID is the ID of the stream the packet belongs to.
StreamID() uint32
// Timestamp is the time the packet was received.
Timestamp() time.Time
// Data is the raw packet data, starting with the IP header.
Data() []byte
}
// PacketCallback is called for each packet received.
// Return false to "unregister" and stop receiving packets.
// It must be safe for concurrent use.
type PacketCallback func(Packet, error) bool
type PacketIO interface {
@@ -39,8 +42,15 @@ type PacketIO interface {
Register(context.Context, PacketCallback) error
// SetVerdict sets the verdict for a packet.
SetVerdict(Packet, Verdict, []byte) error
// ProtectedDialContext is like net.DialContext, but the connection is "protected"
// in the sense that the packets sent/received through the connection must bypass
// the packet IO and not be processed by the callback.
ProtectedDialContext(ctx context.Context, network, address string) (net.Conn, error)
// Close closes the packet IO.
Close() error
// SetCancelFunc gives packet IO access to context cancel function, enabling it to
// trigger a shutdown
SetCancelFunc(cancelFunc context.CancelFunc) error
}
type ErrInvalidPacket struct {

View File

@@ -5,9 +5,12 @@ import (
"encoding/binary"
"errors"
"fmt"
"net"
"os/exec"
"strconv"
"strings"
"syscall"
"time"
"github.com/coreos/go-iptables/iptables"
"github.com/florianl/go-nfqueue"
@@ -27,59 +30,63 @@ const (
nftTable = "opengfw"
)
var nftRulesForward = fmt.Sprintf(`
define ACCEPT_CTMARK=%d
define DROP_CTMARK=%d
define QUEUE_NUM=%d
table %s %s {
chain FORWARD {
type filter hook forward priority filter; policy accept;
ct mark $ACCEPT_CTMARK counter accept
ct mark $DROP_CTMARK counter drop
counter queue num $QUEUE_NUM bypass
}
}
`, nfqueueConnMarkAccept, nfqueueConnMarkDrop, nfqueueNum, nftFamily, nftTable)
var nftRulesLocal = fmt.Sprintf(`
define ACCEPT_CTMARK=%d
define DROP_CTMARK=%d
define QUEUE_NUM=%d
table %s %s {
chain INPUT {
type filter hook input priority filter; policy accept;
ct mark $ACCEPT_CTMARK counter accept
ct mark $DROP_CTMARK counter drop
counter queue num $QUEUE_NUM bypass
}
chain OUTPUT {
type filter hook output priority filter; policy accept;
ct mark $ACCEPT_CTMARK counter accept
ct mark $DROP_CTMARK counter drop
counter queue num $QUEUE_NUM bypass
}
}
`, nfqueueConnMarkAccept, nfqueueConnMarkDrop, nfqueueNum, nftFamily, nftTable)
var iptRulesForward = []iptRule{
{"filter", "FORWARD", []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkAccept), "-j", "ACCEPT"}},
{"filter", "FORWARD", []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkDrop), "-j", "DROP"}},
{"filter", "FORWARD", []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(nfqueueNum), "--queue-bypass"}},
func generateNftRules(local, rst bool) (*nftTableSpec, error) {
if local && rst {
return nil, errors.New("tcp rst is not supported in local mode")
}
table := &nftTableSpec{
Family: nftFamily,
Table: nftTable,
}
table.Defines = append(table.Defines, fmt.Sprintf("define ACCEPT_CTMARK=%d", nfqueueConnMarkAccept))
table.Defines = append(table.Defines, fmt.Sprintf("define DROP_CTMARK=%d", nfqueueConnMarkDrop))
table.Defines = append(table.Defines, fmt.Sprintf("define QUEUE_NUM=%d", nfqueueNum))
if local {
table.Chains = []nftChainSpec{
{Chain: "INPUT", Header: "type filter hook input priority filter; policy accept;"},
{Chain: "OUTPUT", Header: "type filter hook output priority filter; policy accept;"},
}
} else {
table.Chains = []nftChainSpec{
{Chain: "FORWARD", Header: "type filter hook forward priority filter; policy accept;"},
}
}
for i := range table.Chains {
c := &table.Chains[i]
c.Rules = append(c.Rules, "meta mark $ACCEPT_CTMARK ct mark set $ACCEPT_CTMARK") // Bypass protected connections
c.Rules = append(c.Rules, "ct mark $ACCEPT_CTMARK counter accept")
if rst {
c.Rules = append(c.Rules, "ip protocol tcp ct mark $DROP_CTMARK counter reject with tcp reset")
}
c.Rules = append(c.Rules, "ct mark $DROP_CTMARK counter drop")
c.Rules = append(c.Rules, "counter queue num $QUEUE_NUM bypass")
}
return table, nil
}
var iptRulesLocal = []iptRule{
{"filter", "INPUT", []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkAccept), "-j", "ACCEPT"}},
{"filter", "INPUT", []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkDrop), "-j", "DROP"}},
{"filter", "INPUT", []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(nfqueueNum), "--queue-bypass"}},
func generateIptRules(local, rst bool) ([]iptRule, error) {
if local && rst {
return nil, errors.New("tcp rst is not supported in local mode")
}
var chains []string
if local {
chains = []string{"INPUT", "OUTPUT"}
} else {
chains = []string{"FORWARD"}
}
rules := make([]iptRule, 0, 4*len(chains))
for _, chain := range chains {
// Bypass protected connections
rules = append(rules, iptRule{"filter", chain, []string{"-m", "mark", "--mark", strconv.Itoa(nfqueueConnMarkAccept), "-j", "CONNMARK", "--set-mark", strconv.Itoa(nfqueueConnMarkAccept)}})
rules = append(rules, iptRule{"filter", chain, []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkAccept), "-j", "ACCEPT"}})
if rst {
rules = append(rules, iptRule{"filter", chain, []string{"-p", "tcp", "-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkDrop), "-j", "REJECT", "--reject-with", "tcp-reset"}})
}
rules = append(rules, iptRule{"filter", chain, []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkDrop), "-j", "DROP"}})
rules = append(rules, iptRule{"filter", chain, []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(nfqueueNum), "--queue-bypass"}})
}
{"filter", "OUTPUT", []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkAccept), "-j", "ACCEPT"}},
{"filter", "OUTPUT", []string{"-m", "connmark", "--mark", strconv.Itoa(nfqueueConnMarkDrop), "-j", "DROP"}},
{"filter", "OUTPUT", []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(nfqueueNum), "--queue-bypass"}},
return rules, nil
}
var _ PacketIO = (*nfqueuePacketIO)(nil)
@@ -89,16 +96,22 @@ var errNotNFQueuePacket = errors.New("not an NFQueue packet")
type nfqueuePacketIO struct {
n *nfqueue.Nfqueue
local bool
rst bool
rSet bool // whether the nftables/iptables rules have been set
// iptables not nil = use iptables instead of nftables
ipt4 *iptables.IPTables
ipt6 *iptables.IPTables
protectedDialer *net.Dialer
}
type NFQueuePacketIOConfig struct {
QueueSize uint32
Local bool
QueueSize uint32
ReadBuffer int
WriteBuffer int
Local bool
RST bool
}
func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) {
@@ -128,11 +141,38 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) {
if err != nil {
return nil, err
}
if config.ReadBuffer > 0 {
err = n.Con.SetReadBuffer(config.ReadBuffer)
if err != nil {
_ = n.Close()
return nil, err
}
}
if config.WriteBuffer > 0 {
err = n.Con.SetWriteBuffer(config.WriteBuffer)
if err != nil {
_ = n.Close()
return nil, err
}
}
return &nfqueuePacketIO{
n: n,
local: config.Local,
rst: config.RST,
ipt4: ipt4,
ipt6: ipt6,
protectedDialer: &net.Dialer{
Control: func(network, address string, c syscall.RawConn) error {
var err error
cErr := c.Control(func(fd uintptr) {
err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, nfqueueConnMarkAccept)
})
if cErr != nil {
return cErr
}
return err
},
},
}, nil
}
@@ -150,6 +190,12 @@ func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error
streamID: ctIDFromCtBytes(*a.Ct),
data: *a.Payload,
}
// Use timestamp from attribute if available, otherwise use current time as fallback
if a.Timestamp != nil {
p.timestamp = *a.Timestamp
} else {
p.timestamp = time.Now()
}
return okBoolToInt(cb(p, nil))
},
func(e error) int {
@@ -166,9 +212,9 @@ func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error
}
if !n.rSet {
if n.ipt4 != nil {
err = n.setupIpt(n.local, false)
err = n.setupIpt(n.local, n.rst, false)
} else {
err = n.setupNft(n.local, false)
err = n.setupNft(n.local, n.rst, false)
}
if err != nil {
return err
@@ -219,32 +265,39 @@ func (n *nfqueuePacketIO) SetVerdict(p Packet, v Verdict, newPacket []byte) erro
}
}
func (n *nfqueuePacketIO) ProtectedDialContext(ctx context.Context, network, address string) (net.Conn, error) {
return n.protectedDialer.DialContext(ctx, network, address)
}
func (n *nfqueuePacketIO) Close() error {
if n.rSet {
if n.ipt4 != nil {
_ = n.setupIpt(n.local, true)
_ = n.setupIpt(n.local, n.rst, true)
} else {
_ = n.setupNft(n.local, true)
_ = n.setupNft(n.local, n.rst, true)
}
n.rSet = false
}
return n.n.Close()
}
func (n *nfqueuePacketIO) setupNft(local, remove bool) error {
var rules string
if local {
rules = nftRulesLocal
} else {
rules = nftRulesForward
// nfqueue IO does not issue shutdown
func (n *nfqueuePacketIO) SetCancelFunc(cancelFunc context.CancelFunc) error {
return nil
}
func (n *nfqueuePacketIO) setupNft(local, rst, remove bool) error {
rules, err := generateNftRules(local, rst)
if err != nil {
return err
}
var err error
rulesText := rules.String()
if remove {
err = nftDelete(nftFamily, nftTable)
} else {
// Delete first to make sure no leftover rules
_ = nftDelete(nftFamily, nftTable)
err = nftAdd(rules)
err = nftAdd(rulesText)
}
if err != nil {
return err
@@ -252,14 +305,11 @@ func (n *nfqueuePacketIO) setupNft(local, remove bool) error {
return nil
}
func (n *nfqueuePacketIO) setupIpt(local, remove bool) error {
var rules []iptRule
if local {
rules = iptRulesLocal
} else {
rules = iptRulesForward
func (n *nfqueuePacketIO) setupIpt(local, rst, remove bool) error {
rules, err := generateIptRules(local, rst)
if err != nil {
return err
}
var err error
if remove {
err = iptsBatchDeleteIfExists([]*iptables.IPTables{n.ipt4, n.ipt6}, rules)
} else {
@@ -274,15 +324,20 @@ func (n *nfqueuePacketIO) setupIpt(local, remove bool) error {
var _ Packet = (*nfqueuePacket)(nil)
type nfqueuePacket struct {
id uint32
streamID uint32
data []byte
id uint32
streamID uint32
timestamp time.Time
data []byte
}
func (p *nfqueuePacket) StreamID() uint32 {
return p.streamID
}
func (p *nfqueuePacket) Timestamp() time.Time {
return p.timestamp
}
func (p *nfqueuePacket) Data() []byte {
return p.data
}
@@ -314,6 +369,42 @@ func nftDelete(family, table string) error {
return cmd.Run()
}
type nftTableSpec struct {
Defines []string
Family, Table string
Chains []nftChainSpec
}
func (t *nftTableSpec) String() string {
chains := make([]string, 0, len(t.Chains))
for _, c := range t.Chains {
chains = append(chains, c.String())
}
return fmt.Sprintf(`
%s
table %s %s {
%s
}
`, strings.Join(t.Defines, "\n"), t.Family, t.Table, strings.Join(chains, ""))
}
type nftChainSpec struct {
Chain string
Header string
Rules []string
}
func (c *nftChainSpec) String() string {
return fmt.Sprintf(`
chain %s {
%s
%s
}
`, c.Chain, c.Header, strings.Join(c.Rules, "\n\x20\x20\x20\x20"))
}
type iptRule struct {
Table, Chain string
RuleSpec []string

136
io/pcap.go Normal file
View File

@@ -0,0 +1,136 @@
package io
import (
"context"
"hash/crc32"
"io"
"net"
"os"
"sort"
"strings"
"time"
"github.com/google/gopacket"
"github.com/google/gopacket/pcapgo"
)
var _ PacketIO = (*pcapPacketIO)(nil)
type pcapPacketIO struct {
pcapFile io.ReadCloser
pcap *pcapgo.Reader
timeOffset *time.Duration
ioCancel context.CancelFunc
config PcapPacketIOConfig
dialer *net.Dialer
}
type PcapPacketIOConfig struct {
PcapFile string
Realtime bool
}
func NewPcapPacketIO(config PcapPacketIOConfig) (PacketIO, error) {
pcapFile, err := os.Open(config.PcapFile)
if err != nil {
return nil, err
}
handle, err := pcapgo.NewReader(pcapFile)
if err != nil {
return nil, err
}
return &pcapPacketIO{
pcapFile: pcapFile,
pcap: handle,
timeOffset: nil,
ioCancel: nil,
config: config,
dialer: &net.Dialer{},
}, nil
}
func (p *pcapPacketIO) Register(ctx context.Context, cb PacketCallback) error {
go func() {
packetSource := gopacket.NewPacketSource(p.pcap, p.pcap.LinkType())
for packet := range packetSource.Packets() {
p.wait(packet)
networkLayer := packet.NetworkLayer()
if networkLayer != nil {
src, dst := networkLayer.NetworkFlow().Endpoints()
endpoints := []string{src.String(), dst.String()}
sort.Strings(endpoints)
id := crc32.Checksum([]byte(strings.Join(endpoints, ",")), crc32.IEEETable)
cb(&pcapPacket{
streamID: id,
timestamp: packet.Metadata().Timestamp,
data: packet.LinkLayer().LayerPayload(),
}, nil)
}
}
// Give the workers a chance to finish everything
time.Sleep(time.Second)
// Stop the engine when all packets are finished
p.ioCancel()
}()
return nil
}
// A normal dialer is sufficient as pcap IO does not mess up with the networking
func (p *pcapPacketIO) ProtectedDialContext(ctx context.Context, network, address string) (net.Conn, error) {
return p.dialer.DialContext(ctx, network, address)
}
func (p *pcapPacketIO) SetVerdict(pkt Packet, v Verdict, newPacket []byte) error {
return nil
}
func (p *pcapPacketIO) SetCancelFunc(cancelFunc context.CancelFunc) error {
p.ioCancel = cancelFunc
return nil
}
func (p *pcapPacketIO) Close() error {
return p.pcapFile.Close()
}
// Intentionally slow down the replay
// In realtime mode, this is to match the timestamps in the capture
func (p *pcapPacketIO) wait(packet gopacket.Packet) {
if !p.config.Realtime {
return
}
if p.timeOffset == nil {
offset := time.Since(packet.Metadata().Timestamp)
p.timeOffset = &offset
} else {
t := time.Until(packet.Metadata().Timestamp.Add(*p.timeOffset))
time.Sleep(t)
}
}
var _ Packet = (*pcapPacket)(nil)
type pcapPacket struct {
streamID uint32
timestamp time.Time
data []byte
}
func (p *pcapPacket) StreamID() uint32 {
return p.streamID
}
func (p *pcapPacket) Timestamp() time.Time {
return p.timestamp
}
func (p *pcapPacket) Data() []byte {
return p.data
}

View File

@@ -49,7 +49,7 @@ func (l *V2GeoLoader) shouldDownload(filename string) bool {
if os.IsNotExist(err) {
return true
}
dt := time.Now().Sub(info.ModTime())
dt := time.Since(info.ModTime())
if l.UpdateInterval == 0 {
return dt > geoDefaultUpdateInterval
} else {

View File

@@ -14,14 +14,12 @@ type GeoMatcher struct {
ipMatcherLock sync.Mutex
}
func NewGeoMatcher(geoSiteFilename, geoIpFilename string) (*GeoMatcher, error) {
geoLoader := NewDefaultGeoLoader(geoSiteFilename, geoIpFilename)
func NewGeoMatcher(geoSiteFilename, geoIpFilename string) *GeoMatcher {
return &GeoMatcher{
geoLoader: geoLoader,
geoLoader: NewDefaultGeoLoader(geoSiteFilename, geoIpFilename),
geoSiteMatcher: make(map[string]hostMatcher),
geoIpMatcher: make(map[string]hostMatcher),
}, nil
}
}
func (g *GeoMatcher) MatchGeoIp(ip, condition string) bool {

View File

@@ -1,54 +0,0 @@
package v2geo
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadGeoIP(t *testing.T) {
m, err := LoadGeoIP("geoip.dat")
assert.NoError(t, err)
// Exact checks since we know the data.
assert.Len(t, m, 252)
assert.Equal(t, m["cn"].CountryCode, "CN")
assert.Len(t, m["cn"].Cidr, 10407)
assert.Equal(t, m["us"].CountryCode, "US")
assert.Len(t, m["us"].Cidr, 193171)
assert.Equal(t, m["private"].CountryCode, "PRIVATE")
assert.Len(t, m["private"].Cidr, 18)
assert.Contains(t, m["private"].Cidr, &CIDR{
Ip: []byte("\xc0\xa8\x00\x00"),
Prefix: 16,
})
}
func TestLoadGeoSite(t *testing.T) {
m, err := LoadGeoSite("geosite.dat")
assert.NoError(t, err)
// Exact checks since we know the data.
assert.Len(t, m, 1204)
assert.Equal(t, m["netflix"].CountryCode, "NETFLIX")
assert.Len(t, m["netflix"].Domain, 25)
assert.Contains(t, m["netflix"].Domain, &Domain{
Type: Domain_Full,
Value: "netflix.com.edgesuite.net",
})
assert.Contains(t, m["netflix"].Domain, &Domain{
Type: Domain_RootDomain,
Value: "fast.com",
})
assert.Len(t, m["google"].Domain, 1066)
assert.Contains(t, m["google"].Domain, &Domain{
Type: Domain_RootDomain,
Value: "ggpht.cn",
Attribute: []*Domain_Attribute{
{
Key: "cn",
TypedValue: &Domain_Attribute_BoolValue{BoolValue: true},
},
},
})
}

View File

@@ -1,11 +1,15 @@
package ruleset
import (
"context"
"fmt"
"net"
"os"
"reflect"
"strings"
"time"
"github.com/expr-lang/expr/builtin"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/ast"
@@ -16,7 +20,6 @@ import (
"github.com/apernet/OpenGFW/analyzer"
"github.com/apernet/OpenGFW/modifier"
"github.com/apernet/OpenGFW/ruleset/builtins"
"github.com/apernet/OpenGFW/ruleset/builtins/geo"
)
// ExprRule is the external representation of an expression rule.
@@ -55,10 +58,9 @@ type compiledExprRule struct {
var _ Ruleset = (*exprRuleset)(nil)
type exprRuleset struct {
Rules []compiledExprRule
Ans []analyzer.Analyzer
Logger Logger
GeoMatcher *geo.GeoMatcher
Rules []compiledExprRule
Ans []analyzer.Analyzer
Logger Logger
}
func (r *exprRuleset) Analyzers(info StreamInfo) []analyzer.Analyzer {
@@ -100,10 +102,7 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
fullAnMap := analyzersToMap(ans)
fullModMap := modifiersToMap(mods)
depAnMap := make(map[string]analyzer.Analyzer)
geoMatcher, err := geo.NewGeoMatcher(config.GeoSiteFilename, config.GeoIpFilename)
if err != nil {
return nil, err
}
funcMap := buildFunctionMap(config)
// Compile all rules and build a map of analyzers that are used by the rules.
for _, rule := range rules {
if rule.Action == "" && !rule.Log {
@@ -118,13 +117,19 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
action = &a
}
visitor := &idVisitor{Variables: make(map[string]bool), Identifiers: make(map[string]bool)}
patcher := &idPatcher{}
patcher := &idPatcher{FuncMap: funcMap}
program, err := expr.Compile(rule.Expr,
func(c *conf.Config) {
c.Strict = false
c.Expect = reflect.Bool
c.Visitors = append(c.Visitors, visitor, patcher)
registerBuiltinFunctions(c.Functions, geoMatcher)
for name, f := range funcMap {
c.Functions[name] = &builtin.Function{
Name: name,
Func: f.Func,
Types: f.Types,
}
}
},
)
if err != nil {
@@ -138,24 +143,15 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
if isBuiltInAnalyzer(name) || visitor.Variables[name] {
continue
}
// Check if it's one of the built-in functions, and if so,
// skip it as an analyzer & do initialization if necessary.
switch name {
case "geoip":
if err := geoMatcher.LoadGeoIP(); err != nil {
return nil, fmt.Errorf("rule %q failed to load geoip: %w", rule.Name, err)
}
case "geosite":
if err := geoMatcher.LoadGeoSite(); err != nil {
return nil, fmt.Errorf("rule %q failed to load geosite: %w", rule.Name, err)
}
case "cidr":
// No initialization needed for CIDR.
default:
a, ok := fullAnMap[name]
if !ok {
return nil, fmt.Errorf("rule %q uses unknown analyzer %q", rule.Name, name)
if f, ok := funcMap[name]; ok {
// Built-in function, initialize if necessary
if f.InitFunc != nil {
if err := f.InitFunc(); err != nil {
return nil, fmt.Errorf("rule %q failed to initialize function %q: %w", rule.Name, name, err)
}
}
} else if a, ok := fullAnMap[name]; ok {
// Analyzer, add to dependency map
depAnMap[name] = a
}
}
@@ -184,37 +180,12 @@ func CompileExprRules(rules []ExprRule, ans []analyzer.Analyzer, mods []modifier
depAns = append(depAns, a)
}
return &exprRuleset{
Rules: compiledRules,
Ans: depAns,
Logger: config.Logger,
GeoMatcher: geoMatcher,
Rules: compiledRules,
Ans: depAns,
Logger: config.Logger,
}, nil
}
func registerBuiltinFunctions(funcMap map[string]*ast.Function, geoMatcher *geo.GeoMatcher) {
funcMap["geoip"] = &ast.Function{
Name: "geoip",
Func: func(params ...any) (any, error) {
return geoMatcher.MatchGeoIp(params[0].(string), params[1].(string)), nil
},
Types: []reflect.Type{reflect.TypeOf(geoMatcher.MatchGeoIp)},
}
funcMap["geosite"] = &ast.Function{
Name: "geosite",
Func: func(params ...any) (any, error) {
return geoMatcher.MatchGeoSite(params[0].(string), params[1].(string)), nil
},
Types: []reflect.Type{reflect.TypeOf(geoMatcher.MatchGeoSite)},
}
funcMap["cidr"] = &ast.Function{
Name: "cidr",
Func: func(params ...any) (any, error) {
return builtins.MatchCIDR(params[0].(string), params[1].(*net.IPNet)), nil
},
Types: []reflect.Type{reflect.TypeOf((func(string, string) bool)(nil)), reflect.TypeOf(builtins.MatchCIDR)},
}
}
func streamInfoToExprEnv(info StreamInfo) map[string]interface{} {
m := map[string]interface{}{
"id": info.ID,
@@ -299,29 +270,108 @@ func (v *idVisitor) Visit(node *ast.Node) {
// idPatcher patches the AST during expr compilation, replacing certain values with
// their internal representations for better runtime performance.
type idPatcher struct {
Err error
FuncMap map[string]*Function
Err error
}
func (p *idPatcher) Visit(node *ast.Node) {
switch (*node).(type) {
case *ast.CallNode:
callNode := (*node).(*ast.CallNode)
if callNode.Func == nil {
if callNode.Callee == nil {
// Ignore invalid call nodes
return
}
switch callNode.Func.Name {
case "cidr":
cidrStringNode, ok := callNode.Arguments[1].(*ast.StringNode)
if !ok {
return
if f, ok := p.FuncMap[callNode.Callee.String()]; ok {
if f.PatchFunc != nil {
if err := f.PatchFunc(&callNode.Arguments); err != nil {
p.Err = err
return
}
}
cidr, err := builtins.CompileCIDR(cidrStringNode.Value)
if err != nil {
p.Err = err
return
}
callNode.Arguments[1] = &ast.ConstantNode{Value: cidr}
}
}
}
type Function struct {
InitFunc func() error
PatchFunc func(args *[]ast.Node) error
Func func(params ...any) (any, error)
Types []reflect.Type
}
func buildFunctionMap(config *BuiltinConfig) map[string]*Function {
return map[string]*Function{
"geoip": {
InitFunc: config.GeoMatcher.LoadGeoIP,
PatchFunc: nil,
Func: func(params ...any) (any, error) {
return config.GeoMatcher.MatchGeoIp(params[0].(string), params[1].(string)), nil
},
Types: []reflect.Type{reflect.TypeOf(config.GeoMatcher.MatchGeoIp)},
},
"geosite": {
InitFunc: config.GeoMatcher.LoadGeoSite,
PatchFunc: nil,
Func: func(params ...any) (any, error) {
return config.GeoMatcher.MatchGeoSite(params[0].(string), params[1].(string)), nil
},
Types: []reflect.Type{reflect.TypeOf(config.GeoMatcher.MatchGeoSite)},
},
"cidr": {
InitFunc: nil,
PatchFunc: func(args *[]ast.Node) error {
cidrStringNode, ok := (*args)[1].(*ast.StringNode)
if !ok {
return fmt.Errorf("cidr: invalid argument type")
}
cidr, err := builtins.CompileCIDR(cidrStringNode.Value)
if err != nil {
return err
}
(*args)[1] = &ast.ConstantNode{Value: cidr}
return nil
},
Func: func(params ...any) (any, error) {
return builtins.MatchCIDR(params[0].(string), params[1].(*net.IPNet)), nil
},
Types: []reflect.Type{reflect.TypeOf(builtins.MatchCIDR)},
},
"lookup": {
InitFunc: nil,
PatchFunc: func(args *[]ast.Node) error {
var serverStr *ast.StringNode
if len(*args) > 1 {
// Has the optional server argument
var ok bool
serverStr, ok = (*args)[1].(*ast.StringNode)
if !ok {
return fmt.Errorf("lookup: invalid argument type")
}
}
r := &net.Resolver{
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
if serverStr != nil {
address = serverStr.Value
}
return config.ProtectedDialContext(ctx, network, address)
},
}
if len(*args) > 1 {
(*args)[1] = &ast.ConstantNode{Value: r}
} else {
*args = append(*args, &ast.ConstantNode{Value: r})
}
return nil
},
Func: func(params ...any) (any, error) {
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
defer cancel()
return params[1].(*net.Resolver).LookupHost(ctx, params[0].(string))
},
Types: []reflect.Type{
reflect.TypeOf((func(string, *net.Resolver) []string)(nil)),
},
},
}
}

View File

@@ -1,11 +1,13 @@
package ruleset
import (
"context"
"net"
"strconv"
"github.com/apernet/OpenGFW/analyzer"
"github.com/apernet/OpenGFW/modifier"
"github.com/apernet/OpenGFW/ruleset/builtins/geo"
)
type Action int
@@ -100,7 +102,7 @@ type Logger interface {
}
type BuiltinConfig struct {
Logger Logger
GeoSiteFilename string
GeoIpFilename string
Logger Logger
GeoMatcher *geo.GeoMatcher
ProtectedDialContext func(ctx context.Context, network, address string) (net.Conn, error)
}