mirror of
https://github.com/EasyTier/EasyTier.git
synced 2026-09-20 03:22:05 +00:00
Compare commits
29
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
09cd15725e | ||
|
|
90c45d2964 | ||
|
|
0e665eafc6 | ||
|
|
97526a95a1 | ||
|
|
2794b96028 | ||
|
|
42b6d326a8 | ||
|
|
b37c1539e1 | ||
|
|
8312400913 | ||
|
|
19647f296a | ||
|
|
94d056cb78 | ||
|
|
4c7c3d7672 | ||
|
|
d99efba64f | ||
|
|
6d01908593 | ||
|
|
4b654fc56e | ||
|
|
1fdd4b0abe | ||
|
|
ba4fde40ad | ||
|
|
cdec67ff53 | ||
|
|
7e0cdfc683 | ||
|
|
1d80439c7c | ||
|
|
392a970db1 | ||
|
|
368d140b5b | ||
|
|
57cc9922a4 | ||
|
|
4875393327 | ||
|
|
37f742272b | ||
|
|
d4ef9decd8 | ||
|
|
3464cb801a | ||
|
|
e18387b06b | ||
|
|
31c639f70c | ||
|
|
79035ea972 |
@@ -35,7 +35,6 @@ jobs:
|
||||
with:
|
||||
gui: false
|
||||
pnpm: false
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
@@ -244,3 +243,4 @@ jobs:
|
||||
ohpm publish easytier-release.har
|
||||
fi
|
||||
curl --header "Content-Type: application/json" --request POST --data "{}" ${{ secrets.CODEARTS_WEBHOOKS }}
|
||||
|
||||
|
||||
Generated
+144
-117
@@ -129,12 +129,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anes"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.15"
|
||||
@@ -263,6 +257,12 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
|
||||
|
||||
[[package]]
|
||||
name = "ascii"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
|
||||
|
||||
[[package]]
|
||||
name = "async-broadcast"
|
||||
version = "0.7.2"
|
||||
@@ -1145,12 +1145,6 @@ dependencies = [
|
||||
"toml 0.9.12+spec-1.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cast"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.10"
|
||||
@@ -1261,31 +1255,10 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium"
|
||||
version = "0.2.2"
|
||||
name = "chunked_transfer"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"ciborium-ll",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-io"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-ll"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"half",
|
||||
]
|
||||
checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"
|
||||
|
||||
[[package]]
|
||||
name = "cidr"
|
||||
@@ -1632,42 +1605,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
|
||||
dependencies = [
|
||||
"anes",
|
||||
"cast",
|
||||
"ciborium",
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"is-terminal",
|
||||
"itertools 0.10.5",
|
||||
"num-traits",
|
||||
"once_cell",
|
||||
"oorandom",
|
||||
"plotters",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"tinytemplate",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion-plot"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools 0.10.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "critical-section"
|
||||
version = "1.2.0"
|
||||
@@ -2163,6 +2100,16 @@ dependencies = [
|
||||
"dirs-sys 0.5.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-next"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"dirs-sys-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.3.7"
|
||||
@@ -2186,6 +2133,17 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys-next"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users 0.4.5",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dispatch2"
|
||||
version = "0.3.1"
|
||||
@@ -2341,7 +2299,6 @@ dependencies = [
|
||||
"clap_complete",
|
||||
"clap_complete_nushell",
|
||||
"console-subscriber",
|
||||
"criterion",
|
||||
"crossbeam",
|
||||
"ctor 0.8.0",
|
||||
"dashmap",
|
||||
@@ -2365,6 +2322,7 @@ dependencies = [
|
||||
"hickory-resolver",
|
||||
"hickory-server",
|
||||
"hmac",
|
||||
"hotpath",
|
||||
"http",
|
||||
"http_req",
|
||||
"humansize",
|
||||
@@ -2407,6 +2365,7 @@ dependencies = [
|
||||
"quanta",
|
||||
"quinn",
|
||||
"quinn-proto",
|
||||
"quinn-udp",
|
||||
"quote",
|
||||
"rand 0.8.5",
|
||||
"rcgen",
|
||||
@@ -2707,6 +2666,12 @@ version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "encoding"
|
||||
version = "0.2.33"
|
||||
@@ -3965,6 +3930,61 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc2c28b1fa962e433f800ed1ea0bf53dc028d3745cf2acec6cfd28b65ac96afa"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"cfg-if",
|
||||
"crossbeam-channel",
|
||||
"flate2",
|
||||
"flume 0.12.0",
|
||||
"futures-util",
|
||||
"hdrhistogram",
|
||||
"hotpath-macros",
|
||||
"hotpath-meta",
|
||||
"libc",
|
||||
"object",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"prettytable-rs",
|
||||
"quanta",
|
||||
"regex",
|
||||
"rustc-demangle",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tiny_http",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a585238d8daf746e27df0f24d1bbdcd2410e9febff63f9a0173f90d7e71c50f6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros-meta"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "309f63c2f755dead454dd4b3ea8ab5c947f14f8ea435fbcd37fa820e17290e80"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-meta"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68faa91a9e1114dff668cd90560f332da6bbde40dae37ec28ea1c43ca5ce3be3"
|
||||
dependencies = [
|
||||
"hotpath-macros-meta",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.29.1"
|
||||
@@ -5933,6 +5953,15 @@ dependencies = [
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.36.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
@@ -5943,12 +5972,6 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.3.1"
|
||||
@@ -6579,34 +6602,6 @@ dependencies = [
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"plotters-backend",
|
||||
"plotters-svg",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters-backend"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
|
||||
|
||||
[[package]]
|
||||
name = "plotters-svg"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
|
||||
dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pnet"
|
||||
version = "0.35.0"
|
||||
@@ -6837,6 +6832,19 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prettytable-rs"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"is-terminal",
|
||||
"lazy_static",
|
||||
"term",
|
||||
"unicode-width 0.1.11",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "primeorder"
|
||||
version = "0.13.6"
|
||||
@@ -7896,6 +7904,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.0"
|
||||
@@ -9820,6 +9834,17 @@ dependencies = [
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "term"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
|
||||
dependencies = [
|
||||
"dirs-next",
|
||||
"rustversion",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "terminal_size"
|
||||
version = "0.4.1"
|
||||
@@ -9975,6 +10000,18 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "825f6c8a18bc36d56a62f66af7296385b628c9c5543a8663d4c217fc920bfefd"
|
||||
|
||||
[[package]]
|
||||
name = "tiny_http"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82"
|
||||
dependencies = [
|
||||
"ascii",
|
||||
"chunked_transfer",
|
||||
"httpdate",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.7.6"
|
||||
@@ -9985,16 +10022,6 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.8.0"
|
||||
|
||||
@@ -27,3 +27,9 @@ lto = true
|
||||
codegen-units = 1
|
||||
opt-level = 3
|
||||
strip = true
|
||||
|
||||
# For hotpath CPU profiling: samply needs debug symbols and unstripped binaries.
|
||||
[profile.hotpath]
|
||||
inherits = "release"
|
||||
strip = false
|
||||
debug = "line-tables-only"
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
# hotpath CPU 热点分析与发包链路优化
|
||||
|
||||
## 概述
|
||||
|
||||
本文档记录了使用 hotpath + samply 对 easytier-core 发包链路进行 CPU 热点分析的全过程,包括工具链搭建、热点定位、优化实施、踩坑记录和最终 benchmark 结果。
|
||||
|
||||
## 最终 benchmark 数据
|
||||
|
||||
### 真实性能对比(不带 hotpath,3 runs average)
|
||||
|
||||
origin/main baseline 使用 `git worktree` 从 origin/main 构建,仅添加 bench example +
|
||||
loopback bind fix(TCP/UDP convergence 需要)。无任何优化代码。
|
||||
|
||||
| Tunnel | origin/main baseline | 优化后 | **提升** | 带宽(优化后) |
|
||||
| -------- | -------------------- | -------------- | --------- | -------------- |
|
||||
| **Ring** | 293K pps / 3.3 Gbps | **1,124K pps** | **+284%** | 12.6 Gbps |
|
||||
| **TCP** | 298K pps / 3.3 Gbps | **975K pps** | **+227%** | 10.9 Gbps |
|
||||
| **UDP** | 630K pps / 7.1 Gbps | **1,066K pps** | **+69%** | 11.9 Gbps |
|
||||
|
||||
UDP baseline 本身较高(630K vs 293K/298K),因为 UDP tunnel 的 forward_from_ring_to_udp
|
||||
独立 task 提供了天然的 pipeline overlap,部分隐藏了 channel 开销。
|
||||
|
||||
### 带 hotpath profiling(timing 可见,但有 observer effect)
|
||||
|
||||
| Tunnel | 原始 pps | 优化后 pps | 提升 | MpscTunnelSender::send |
|
||||
| ------ | -------- | ---------- | ----- | ---------------------- |
|
||||
| Ring | 234K | 478K | +104% | 138ns (原 2.23µs) |
|
||||
| UDP | N/A | 440K | — | 294ns |
|
||||
| TCP | N/A | 453K | — | 378ns |
|
||||
|
||||
### hotpath observer effect
|
||||
|
||||
**hotpath 测量基础设施引入了 ~54-57% 的性能开销:**
|
||||
|
||||
| Tunnel | 不带 hotpath | 带 hotpath | hotpath 开销 |
|
||||
| ------ | ------------ | ---------- | ------------ |
|
||||
| Ring | 1,124K pps | 478K pps | **-57%** |
|
||||
| TCP | 975K pps | 453K pps | **-54%** |
|
||||
|
||||
**含义:**
|
||||
|
||||
- timing 数据里的 `send_msg_by_ip: 2.15µs` 是膨胀值,真实成本 ~0.9µs
|
||||
- 所有 timing 数据需要按 ~2.3x 校准才能反映真实开销
|
||||
- hotpath 适用于相对比较(优化前 vs 后),不适用于绝对性能评估
|
||||
- 生产环境部署不应用 hotpath feature 编译
|
||||
|
||||
测试条件:4 threads, 1400B packets, 10s, 宿主机直跑。
|
||||
|
||||
---
|
||||
|
||||
## 工具链搭建
|
||||
|
||||
### hotpath + samply 安装
|
||||
|
||||
```bash
|
||||
# hotpath TUI (console)
|
||||
cargo install hotpath --version 0.18.0 --bin hotpath --features tui
|
||||
|
||||
# hotpath-samply (samply wrapper,autospawn 依赖)
|
||||
cargo install hotpath --version 0.18.0 --bin hotpath-samply
|
||||
|
||||
# samply 本体(hotpath-samply 内部 spawn samply record)
|
||||
cargo install samply
|
||||
```
|
||||
|
||||
### 内核参数
|
||||
|
||||
samply 需要 perf_event 开销,需要调整内核参数:
|
||||
|
||||
```bash
|
||||
echo '1' | sudo tee /proc/sys/kernel/perf_event_paranoid
|
||||
echo '65536' | sudo tee /proc/sys/kernel/perf_event_mlock_kb
|
||||
```
|
||||
|
||||
- `perf_event_paranoid` 默认 2(不允许非 root 采样),需降到 1。
|
||||
- `perf_event_mlock_kb` 默认 516 KB,32 核机器上 samply 的 mmap buffer 总量超限,需增大到 65536。
|
||||
|
||||
### hotpath profile 编译
|
||||
|
||||
```toml
|
||||
# Cargo.toml
|
||||
[profile.hotpath]
|
||||
inherits = "release"
|
||||
strip = false
|
||||
debug = "line-tables-only"
|
||||
```
|
||||
|
||||
samply 需要 debug symbols 且不能 strip。release profile 默认 `strip = true`,必须用单独的 profile。
|
||||
|
||||
### Docker 隔离环境(可选,TCP/UDP bench)
|
||||
|
||||
修复 loopback bind 地址后(见坑 11),TCP/UDP bench 可以直接在宿主机上跑,不需要 Docker:
|
||||
|
||||
```bash
|
||||
# Ring(进程内,无需隔离)
|
||||
HOTPATH_TUNNEL=ring ./target/hotpath/examples/cpu_hotspot_ring
|
||||
|
||||
# TCP/UDP(修复后也支持宿主机直跑)
|
||||
HOTPATH_TUNNEL=tcp ./target/hotpath/examples/cpu_hotspot_ring
|
||||
```
|
||||
|
||||
如果仍有 convergence 问题(多网卡环境),用 Docker 提供独立 netns:
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v "$(pwd)/target/hotpath/examples/cpu_hotspot_ring:/bench:ro" \
|
||||
-e HOTPATH_TUNNEL=tcp \
|
||||
-e HOTPATH_BENCH_SECS=10 \
|
||||
fedora:latest \
|
||||
/bench
|
||||
```
|
||||
|
||||
Docker 镜像需要匹配宿主机的 glibc 版本。Fedora 宿主用 `fedora:latest`。
|
||||
|
||||
---
|
||||
|
||||
## 踩坑记录
|
||||
|
||||
### 坑 1:samply 报 "failed to spawn samply: No such file or directory"
|
||||
|
||||
**现象**:hotpath CPU report 显示 `failed to spawn samply: No such file or directory (os error 2)`
|
||||
|
||||
**原因**:hotpath-samply 只是 wrapper,它内部 spawn `samply record --pid <pid>` 来采集 CPU 样本。samply 本体没装。
|
||||
|
||||
**解决**:
|
||||
|
||||
```bash
|
||||
cargo install samply
|
||||
```
|
||||
|
||||
如果 autospawn 找不到 hotpath-samply 本身,用环境变量指定完整路径:
|
||||
|
||||
```bash
|
||||
export HOTPATH_SAMPLY_WRAPPER_BIN=~/.cargo/bin/hotpath-samply
|
||||
```
|
||||
|
||||
### 坑 2:samply 报 "Failed to start profiling: mmap failed"
|
||||
|
||||
**现象**:samply 启动后立即报 mmap 失败。
|
||||
|
||||
**原因**:`perf_event_mlock_kb` 默认只有 516 KB。32 核机器上 samply 为每个 CPU core 创建 mmap buffer,总 mmap 量超过限制。
|
||||
|
||||
**解决**:
|
||||
|
||||
```bash
|
||||
echo '65536' | sudo tee /proc/sys/kernel/perf_event_mlock_kb
|
||||
```
|
||||
|
||||
### 坑 3:samply 报 "samply exited with status exit status: 1"
|
||||
|
||||
**现象**:samply 被 spawn 了但 exit 1。
|
||||
|
||||
**原因**:同坑 2——`perf_event_paranoid = 2` 时非 root 用户无法使用 perf_event_open。
|
||||
|
||||
**解决**:
|
||||
|
||||
```bash
|
||||
echo '1' | sudo tee /proc/sys/kernel/perf_event_paranoid
|
||||
```
|
||||
|
||||
### 坑 4:火焰图全是地址,看不到符号
|
||||
|
||||
**现象**:samply profile 打开后火焰图全是 `0x31dd24` 之类的地址。
|
||||
|
||||
**原因**:samply profile 里存储的是地址(不内联符号化)。符号化在查看时通过 symbol server 动态完成。如果直接下载 raw JSON 上传到 profiler.firefox.com,符号 server 无法访问本地二进制文件。
|
||||
|
||||
**解决**:必须用 `samply load` 本地打开(它启动 symbol server 自动做符号化):
|
||||
|
||||
```bash
|
||||
samply load /tmp/hotpath/<session>/hp.json.gz
|
||||
```
|
||||
|
||||
不要下载 JSON 再上传到 profiler.firefox.com。
|
||||
|
||||
### 坑 5:samply 符号化后 `_dl_mcount_wrapper` 占 18.1%
|
||||
|
||||
**现象**:send_msg_internal inclusive 分析显示 `_dl_mcount_wrapper` 占 18.1% CPU。
|
||||
|
||||
**原因**:nm 的动态符号表里 `_dl_mcount_wrapper`(0x1498d0)到下一个符号(0x1b3e9e)之间有 **425 KB gap**。nm 的 bisect 查找把 gap 内所有地址错误归因到 `_dl_mcount_wrapper`。gap 里实际是 AVX2 优化的 memmove/memcmp/memset 等函数。
|
||||
|
||||
**解决**:用 addr2line 精确解析(而非 nm bisect)。实际开销是 memmove 1.67% + memcmp 0.20% + memset 0.19% = 2.1%,不是 18%。**没有 profiling 钩子**。
|
||||
|
||||
### 坑 6:parking_lot::MutexGuard 不是 Send
|
||||
|
||||
**现象**:使用 `parking_lot::Mutex` 替代 `tokio::sync::Mutex` 后,编译报 31 个 "future cannot be sent between threads safely"。
|
||||
|
||||
**原因**:`parking_lot::MutexGuard` 刻意不实现 `Send`——锁必须在获取它的同一个线程上释放。在 async fn 里 guard 跨 await 点会导致 Future 不是 Send,tokio multi_thread runtime 拒绝 spawn。
|
||||
|
||||
**解决**:自定义 `SpinSink`(AtomicBool spinlock),`SpinGuard` 只持有 `&SpinSink` 引用(SpinSink: Sync via unsafe impl),是 Send。
|
||||
|
||||
### 坑 7:std::sync::MutexGuard 也不是 Send(在某些配置下)
|
||||
|
||||
**现象**:`std::sync::Mutex` 同样报 "future cannot be sent between threads safely"。
|
||||
|
||||
**原因**:Rust 标准库的 `MutexGuard` 的 Send 实现依赖于内部类型。`Pin<Box<dyn ZCPacketSink>>` 包含 trait object,某些配置下 guard 不是 Send。
|
||||
|
||||
**解决**:用自定义 SpinSink 绕过所有标准 Mutex 实现。
|
||||
|
||||
### 坑 8:direct sink path 没有性能提升
|
||||
|
||||
**现象**:去掉 channel 中转(MpscTunnelSender 直接持有 sink),从 3 个 await 点(lock + feed + flush)改为 try_lock + poll_fn 合并。MpscTunnelSender::send 仍然 ~2µs。
|
||||
|
||||
**原因**:瓶颈不在 lock 或 channel,而在 **async fn Future 状态机的固有开销**。每次 `.await` 创建一个 Future struct、poll 它、drop 它。即使 poll 立即返回 Ready,整个 async machinery 开销 ~2µs。RingSink 实际操作只有 ~40ns(2%)。
|
||||
|
||||
**解决**:用 `noop_waker()` 在 async fn 内部同步调用 Sink trait 方法(poll_ready + start_send + poll_flush)。async fn 在第一次 poll 就同步完成返回——绕过所有 async 调度开销。开销从 2µs 降到 ~140ns。
|
||||
|
||||
### 坑 9:sync send 破坏了 TCP/UDP tunnel
|
||||
|
||||
**现象**:把 `send` 从 `async fn` 改为 sync `fn` 后,所有 TCP/UDP 相关测试失败(452 个失败)。
|
||||
|
||||
**原因**:TCP/UDP tunnel 用 channel mode(`MpscTunnel::new`)。sync `send` 的 channel path 只做 `try_send`,channel 满时返回 `BufferFull`(丢包),而不是 `send().await`(等待背压)。丢包导致 TCP/UDP 连接握手失败。
|
||||
|
||||
**解决**:保持 `send` 为 async fn。direct path(ring/UDP/TCP)内部用 noop_waker 同步完成(不 yield)。channel path 仍然走 async `send_async().await`。async fn wrapper 对 direct path 只有 ~100ns 开销(Future struct 创建 + 单次 poll),因为不 yield。
|
||||
|
||||
### 坑 10:poll_flush Pending 返回 Shutdown 导致连接断开
|
||||
|
||||
**现象**:noop_waker 模式下,TCP tunnel 的 `poll_flush` 可能返回 Pending(TCP 写缓冲区满)。返回 `Err(Shutdown)` 导致 PeerConn 认为连接断开。
|
||||
|
||||
**原因**:TCP 的 `FramedWriter::poll_flush` 做实际 socket write(系统调用)。socket 缓冲区满时返回 Pending。数据已经在 BufList 里,不需要 panic。
|
||||
|
||||
**解决**:poll_flush Pending 时返回 `Ok(())`。数据已在 buffer(ring buffer 或 BufList),后续操作会消费它。Pending 只意味着 "还没 flush 到网络",不是 "错误"。
|
||||
|
||||
### 坑 11:TCP/UDP bench convergence 失败
|
||||
|
||||
**现象**:TCP/UDP tunnel 的 bench 中,两个实例无法建立连接(routes did not converge within 15s)。
|
||||
|
||||
**原因**:`set_bind_addr_for_peer_connector`(connector/mod.rs:70-77)收集所有本机 IP 作为 TCP bind 地址,但不包含 `127.0.0.1`。connector 绑定到 `172.17.0.2`(Docker eth0)后连接 `127.0.0.1` 路由不通 → 2 秒超时。
|
||||
|
||||
**解决**:在 bind 地址列表头部加入 `127.0.0.1:0`。connector 遍历所有 bind 地址,loopback 先被尝试,localhost 连接成功。
|
||||
|
||||
### 坑 12:ShardedCounter (#2385) 在高频路径引入回退
|
||||
|
||||
**现象**:cherry-pick PR #2385(ShardedCounter 替代 UnsafeCell)后,pps 下降 17%(246K → 203K)。
|
||||
|
||||
**原因**:ShardedCounter 的 TLS 分片设计优化多线程 contention,但每包调用 16 次 `ShardedCounter::add`(TLS load + store),单次 ~14ns,总 224ns/包。比原来的 `UnsafeCell`(~2ns/次)高 6 倍。每包 16 次的调用频率让 TLS 开销累积。
|
||||
|
||||
**教训**:TLS 分片策略适合 **低频高并发** 场景,不适合 **高频单线程** 的发包热路径。
|
||||
|
||||
### 坑 13:ZCPacket pool 不如 glibc tcache
|
||||
|
||||
**现象**:用 `crossbeam_queue::ArrayQueue` 做 BytesMut 对象池,每包从池取/归还。性能没有提升(甚至 -15%)。
|
||||
|
||||
**原因**:glibc malloc 对 ~1500 bytes 小块分配有 thread-local cache(tcache),单次 alloc ~10-15ns。ArrayQueue 的 pop/push 是 CAS 操作(~20-40ns),比 tcache 更慢。pool 还多了 capacity 检查和 clear 操作。
|
||||
|
||||
**教训**:手动对象池在现代 glibc tcache 面前没有优势。真正需要 pool 的场景是避免 munmap(大块 >128KB 分配),不是小块。
|
||||
|
||||
### 坑 14:Pipeline (FuturesUnordered) 效果微小
|
||||
|
||||
**现象**:用 FuturesUnordered 让多个 send_msg_by_ip 并发(pipeline_depth=4),pps 只提升 1.6%。
|
||||
|
||||
**原因**:try_send fast path 让 MpscTunnelSender::send 立即返回(不 await)。多个 send_msg_by_ip 之间没有自然的时间重叠——它们在 CPU 上是串行的。pipeline 需要利用 await 等待时间,但 fast path 消除了 await。
|
||||
|
||||
### 坑 15:hotpath 测量引入 54% observer effect
|
||||
|
||||
**现象**:同一 binary 带 hotpath feature 和不带 hotpath feature 跑 bench,pps 差距巨大。
|
||||
|
||||
**数据**:
|
||||
|
||||
| Tunnel | 不带 hotpath | 带 hotpath | hotpath 开销 |
|
||||
| ------ | ------------ | ---------- | ------------ |
|
||||
| Ring | 1,124K pps | 478K pps | **-57%** |
|
||||
| TCP | 975K pps | 453K pps | **-54%** |
|
||||
|
||||
**原因**:hotpath `#[measure]` / `#[measure_all]` 在每个标注的 async fn 上包装 Future struct,每次 poll 记录开始/结束时间(quanta::Instant ~5ns × 2)、更新统计(atomic 操作)。measure_all 覆盖的 impl 块内所有方法都被插桩。当有 ~30 个 measure 点在发包热路径上时,累计开销超过 50%。
|
||||
|
||||
**教训**:
|
||||
|
||||
- hotpath timing 数据**适用于相对比较**(优化前 vs 后),**不适用于绝对性能评估**
|
||||
- 生产环境**不应**用 hotpath feature 编译
|
||||
- 要获取真实 pps,编译不带 `--features hotpath` 的版本
|
||||
- timing 数据按 ~2.3x 校准可近似真实开销
|
||||
|
||||
---
|
||||
|
||||
## 优化实施记录
|
||||
|
||||
### 真实提升(不带 hotpath,origin/main baseline 对比)
|
||||
|
||||
baseline 构建:`git worktree` 从 origin/main,仅添加 bench example + loopback bind fix。
|
||||
|
||||
| Tunnel | baseline | 优化后 | 提升 |
|
||||
| ------ | -------- | -------------- | --------- |
|
||||
| Ring | 293K pps | **1,124K pps** | **+284%** |
|
||||
| TCP | 298K pps | **975K pps** | **+227%** |
|
||||
| UDP | 630K pps | **1,066K pps** | **+69%** |
|
||||
|
||||
### 有效优化(按贡献排序)
|
||||
|
||||
| 优化 | 带 hotpath pps 变化 | 真实提升来源 | 机制 |
|
||||
| ------------------------ | ------------------- | ------------------------ | ---------------------------------------------------------- |
|
||||
| **noop_waker sync send** | **+90%** | **核心突破** | RingSink/FramedWriter 直接 sync poll,绕过 async machinery |
|
||||
| try_send fast path | +7% | 次要 | 跳过 tokio mpsc semaphore |
|
||||
| #2385 ZCPacket safe init | +5% (TCP) | TCP 专属 | copy_nonoverlapping 无 aliasing 检查 |
|
||||
| metrics batch + sync | +1.6% | 小幅 | batch CounterHandle + sync fast path |
|
||||
| #2381 advance (零拷贝) | ~0% | 代码质量 | Buf::advance 消除 split_off Arc churn |
|
||||
| channel 32→1024 | ~0% | 减少 fallback | 更大 buffer |
|
||||
| 接收侧 try_recv | ~0% (单向) | 双向有价值 | 消除 recv().await async overhead |
|
||||
| loopback bind fix | — | TCP/UDP convergence 必需 | 127.0.0.1 加入 bind 地址列表 |
|
||||
|
||||
### 验证无效并回退
|
||||
|
||||
| 尝试 | 结果 | 原因 |
|
||||
| ---------------------------------- | -------- | ----------------------------------- |
|
||||
| ShardedCounter (#2385) | -17% pps | TLS 分片高频开销 > UnsafeCell |
|
||||
| ZCPacket pool | -15% pps | glibc tcache 比 ArrayQueue CAS 更快 |
|
||||
| Allocator 切换 (jemalloc/mimalloc) | ~0% | 小块分配 tcache 都已足够 |
|
||||
| Pipeline (FuturesUnordered) | +1.6% | try_send 消除了 await 空隙 |
|
||||
| dashmap 合并 | ~0% | contains_key 本身 ~50ns |
|
||||
|
||||
### noop_waker 技术详解
|
||||
|
||||
核心原理:async fn `send()` 内部用 `noop_waker()` 构造 dummy Context,直接调 Sink trait 的 `poll_ready` + `start_send` + `poll_flush`。RingSink 在 ring buffer 不满时所有操作立即返回 Ready——noop_waker 永远不会被触发。
|
||||
|
||||
```rust
|
||||
pub async fn send(&self, item: ZCPacket) -> Result<(), TunnelError> {
|
||||
if let Some(sink) = &self.direct_sink {
|
||||
if let Some(mut guard) = sink.try_lock() {
|
||||
let waker = futures::task::noop_waker();
|
||||
let mut cx = std::task::Context::from_waker(&waker);
|
||||
match guard.as_mut().poll_ready(&mut cx) {
|
||||
Poll::Ready(Ok(())) => {
|
||||
guard.as_mut().start_send(item)?;
|
||||
match guard.as_mut().poll_flush(&mut cx) {
|
||||
Poll::Ready(Err(e)) => return Err(e),
|
||||
_ => return Ok(()), // Ready(Ok) 或 Pending 都返回 Ok
|
||||
}
|
||||
}
|
||||
// ...
|
||||
}
|
||||
}
|
||||
return Err(TunnelError::BufferFull);
|
||||
}
|
||||
// Channel mode: async with backpressure
|
||||
self.send_async(item).await
|
||||
}
|
||||
```
|
||||
|
||||
**为什么 Pending 返回 Ok**:poll_flush Pending 意味着数据已在 buffer(ring buffer 或 BufList)但还没 flush 到网络。forward task 或下一次 send 会消费它。这是安全的——数据不丢、不乱序。
|
||||
|
||||
**适用范围**:所有 Sink 的 `start_send` 是同步内存操作的 tunnel:
|
||||
|
||||
- Ring tunnel: RingSink → ring buffer(内存)
|
||||
- UDP tunnel: RingSink → ring buffer → forward_from_ring_to_udp task → socket
|
||||
- TCP tunnel: FramedWriter → BufList(内存)→ poll_flush 时 write socket
|
||||
|
||||
---
|
||||
|
||||
## hotpath measure 布点
|
||||
|
||||
### 当前覆盖
|
||||
|
||||
```
|
||||
send_msg_by_ip ✅ measure
|
||||
├─ try_compress_and_encrypt ✅ measure
|
||||
├─ get_msg_dst_peer_ipv4 ✅ measure
|
||||
├─ run_nic_packet_process_pipeline ✅ measure
|
||||
├─ send_msg_internal ✅ measure
|
||||
│ ├─ PeerMap::send_msg_directly ✅ measure_all
|
||||
│ ├─ PeerMap::get_peer_by_id ✅ measure_all
|
||||
│ ├─ PeerMap::get_gateway_peer_id ✅ measure_all
|
||||
│ ├─ PeerMap::has_peer ✅ measure_all
|
||||
│ ├─ record_tx_fast ❌ (sync fn, 无 measure)
|
||||
│ └─ Peer::send_msg ✅ measure
|
||||
│ └─ PeerConn::send_msg ✅ measure
|
||||
│ └─ MpscTunnelSender::send ✅ measure
|
||||
├─ MpscTunnel::forward_one_round ✅ measure
|
||||
│ ├─ RingSink::poll_ready ✅ measure_all
|
||||
│ ├─ RingSink::start_send ✅ measure_all
|
||||
│ └─ RingSink::poll_flush ✅ measure_all
|
||||
└─ CidrSet::* ✅ measure_all
|
||||
```
|
||||
|
||||
### 布点排除项(避免与已有 PR 冲突)
|
||||
|
||||
| 文件 | 排除原因 |
|
||||
| ---------------------- | -------------------- |
|
||||
| stats_manager.rs | PR #2385 重写中 |
|
||||
| traffic_metrics.rs | 依赖 stats_manager |
|
||||
| peer_manager.rs (部分) | advisor/001-002 改动 |
|
||||
| peer_conn.rs (部分) | advisor/001-002 改动 |
|
||||
| tunnel/mpsc.rs (部分) | perf/001 改动 |
|
||||
| packet_def.rs | perf/001-003 改动 |
|
||||
| peer_ospf_route.rs | advisor/003-004 改动 |
|
||||
|
||||
---
|
||||
|
||||
## 运行方式
|
||||
|
||||
### Ring tunnel bench
|
||||
|
||||
```bash
|
||||
cargo build --profile hotpath --features hotpath --example cpu_hotspot_ring
|
||||
HOTPATH_BENCH_SECS=15 ./target/hotpath/examples/cpu_hotspot_ring
|
||||
```
|
||||
|
||||
### TCP/UDP bench(需要 Docker 隔离)
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v "$(pwd)/target/hotpath/examples/cpu_hotspot_ring:/bench:ro" \
|
||||
-e HOTPATH_TUNNEL=tcp \
|
||||
-e HOTPATH_BENCH_SECS=10 \
|
||||
fedora:latest \
|
||||
/bench
|
||||
```
|
||||
|
||||
### 带 samply CPU profiling
|
||||
|
||||
```bash
|
||||
export PATH=$HOME/.cargo/bin:$PATH
|
||||
cargo run --profile hotpath --features hotpath,hotpath-cpu --example cpu_hotspot_ring
|
||||
|
||||
# 另一终端查看 CPU top
|
||||
hotpath console
|
||||
```
|
||||
|
||||
### 环境变量
|
||||
|
||||
| 变量 | 默认 | 说明 |
|
||||
| ---------------------------- | ---- | ----------------------- |
|
||||
| `HOTPATH_BENCH_SECS` | 30 | 打流持续秒数 |
|
||||
| `HOTPATH_PKT_SIZE` | 1400 | 包大小 |
|
||||
| `HOTPATH_TUNNEL` | ring | ring / udp / tcp |
|
||||
| `HOTPATH_PIPELINE` | 1 | pipeline 深度 |
|
||||
| `HOTPATH_SAMPLY_WRAPPER_BIN` | — | hotpath-samply 完整路径 |
|
||||
| `HOTPATH_SAMPLY_BIN` | — | samply 本体完整路径 |
|
||||
@@ -163,10 +163,14 @@ async function registerVpnServiceListener() {
|
||||
)
|
||||
}
|
||||
|
||||
function getRoutesForVpn(routes: Route[] | undefined, node_config: NetworkTypes.NetworkConfig): string[] {
|
||||
function getRoutesForVpn(routes: Route[], node_config: NetworkTypes.NetworkConfig): string[] {
|
||||
if (!routes) {
|
||||
return []
|
||||
}
|
||||
|
||||
const ret = []
|
||||
for (const r of routes ?? []) {
|
||||
for (let cidr of r.proxy_cidrs ?? []) {
|
||||
for (const r of routes) {
|
||||
for (let cidr of r.proxy_cidrs) {
|
||||
if (!cidr.includes('/')) {
|
||||
cidr += '/32'
|
||||
}
|
||||
@@ -174,9 +178,9 @@ function getRoutesForVpn(routes: Route[] | undefined, node_config: NetworkTypes.
|
||||
}
|
||||
}
|
||||
|
||||
for (const route of node_config.routes ?? []) {
|
||||
ret.push(route)
|
||||
}
|
||||
node_config.routes.forEach(r => {
|
||||
ret.push(r)
|
||||
})
|
||||
|
||||
if (node_config.enable_magic_dns) {
|
||||
ret.push('100.100.100.101/32')
|
||||
@@ -211,15 +215,14 @@ export async function onNetworkInstanceChange(instanceId: string) {
|
||||
console.log('vpn service skipped because no_tun is enabled', instanceId)
|
||||
return
|
||||
}
|
||||
const curNetworkInfo = (await collectNetworkInfo(instanceId))?.info?.map?.[instanceId]
|
||||
const curNetworkInfo = (await collectNetworkInfo(instanceId)).info.map[instanceId]
|
||||
if (!curNetworkInfo || curNetworkInfo?.error_msg?.length) {
|
||||
console.warn('vpn service skipped because network info is unavailable', instanceId, curNetworkInfo?.error_msg)
|
||||
await doStopVpn()
|
||||
return
|
||||
}
|
||||
|
||||
const virtualIpv4 = curNetworkInfo.my_node_info?.virtual_ipv4
|
||||
const virtual_ip = virtualIpv4?.address?.addr ? Utils.ipv4ToString(virtualIpv4.address) : undefined
|
||||
const virtual_ip = Utils.ipv4ToString(curNetworkInfo?.my_node_info?.virtual_ipv4.address)
|
||||
|
||||
if (config.dhcp && (!virtual_ip || !virtual_ip.length)) {
|
||||
console.log('DHCP enabled but no IP yet, will retry in', DHCP_POLLING_INTERVAL, 'ms')
|
||||
@@ -234,7 +237,7 @@ export async function onNetworkInstanceChange(instanceId: string) {
|
||||
return
|
||||
}
|
||||
|
||||
let network_length = virtualIpv4?.network_length
|
||||
let network_length = curNetworkInfo?.my_node_info?.virtual_ipv4.network_length
|
||||
if (!network_length) {
|
||||
network_length = 24
|
||||
}
|
||||
@@ -287,7 +290,7 @@ async function isNoTunEnabled(instanceId: string | undefined) {
|
||||
|
||||
async function findRunningTunInstanceId() {
|
||||
const instanceIds = await listNetworkInstanceIds()
|
||||
const runningIds = (instanceIds.running_inst_ids ?? []).map(Utils.UuidToStr)
|
||||
const runningIds = instanceIds.running_inst_ids.map(Utils.UuidToStr)
|
||||
console.log('vpn service sync running instances', JSON.stringify(runningIds))
|
||||
|
||||
for (const instanceId of runningIds) {
|
||||
|
||||
@@ -9,7 +9,7 @@ export class GUIRemoteClient implements Api.RemoteClient {
|
||||
await backend.runNetworkInstance(config, save);
|
||||
}
|
||||
async get_network_info(inst_id: string): Promise<NetworkTypes.NetworkInstanceRunningInfo | undefined> {
|
||||
return backend.collectNetworkInfo(inst_id).then(infos => infos.info?.map?.[inst_id]);
|
||||
return backend.collectNetworkInfo(inst_id).then(infos => infos.info.map[inst_id]);
|
||||
}
|
||||
async list_network_instance_ids(): Promise<Api.ListNetworkInstanceIdResponse> {
|
||||
return backend.listNetworkInstanceIds();
|
||||
@@ -44,4 +44,4 @@ export class GUIRemoteClient implements Api.RemoteClient {
|
||||
return await backend.getNetworkMetas(instance_ids);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -114,12 +114,6 @@ function allFieldFixture() {
|
||||
networking_method: NetworkingMethod.Manual,
|
||||
public_server_url: 'tcp://public.example:11010',
|
||||
peer_urls: [' tcp://peer-a:11010 ', '', 'udp://peer-b:11010'],
|
||||
peers: [
|
||||
{
|
||||
uri: 'tcp://peer-a:11010',
|
||||
peer_public_key: 'peer-a-public-key',
|
||||
},
|
||||
],
|
||||
proxy_cidrs: ['10.10.0.0/16', '192.168.2.0/24->10.99.0.0/24'],
|
||||
enable_vpn_portal: true,
|
||||
vpn_portal_listen_port: 23000,
|
||||
@@ -265,8 +259,6 @@ function assertFullFieldRoundTrip() {
|
||||
assert.equal(backend.networking_method, 'Manual')
|
||||
assert.equal(backend.public_server_url, '')
|
||||
assert.deepEqual(backend.peer_urls, ['tcp://peer-a:11010', 'udp://peer-b:11010'])
|
||||
assert.equal(backend.peers[0].peer_public_key, 'peer-a-public-key')
|
||||
assert.deepEqual(backend.peers[1], { uri: 'udp://peer-b:11010' })
|
||||
assert.equal(backend.data_compress_algo, 'Zstd')
|
||||
assert.equal(backend.instance_recv_bps_limit, '9007199254740993')
|
||||
assert.equal(backend.secure_mode.enabled, true)
|
||||
@@ -423,59 +415,6 @@ function assertNetworkingMethodNormalization() {
|
||||
})
|
||||
|
||||
assert.deepEqual(missing.peer_urls, ['tcp://one', 'udp://two'])
|
||||
|
||||
const publicServerMissingUrl = normalizeNetworkConfig({
|
||||
...DEFAULT_NETWORK_CONFIG(),
|
||||
networking_method: 'PublicServer',
|
||||
public_server_url: '',
|
||||
peer_urls: ['tcp://manual.example:11010'],
|
||||
})
|
||||
|
||||
assert.deepEqual(publicServerMissingUrl.peer_urls, [])
|
||||
}
|
||||
|
||||
function assertPeerPublicKeysPreserved() {
|
||||
const normalized = normalizeNetworkConfig({
|
||||
...DEFAULT_NETWORK_CONFIG(),
|
||||
peer_urls: [],
|
||||
peers: [
|
||||
{
|
||||
uri: ' tcp://peer-a:11010 ',
|
||||
peer_public_key: 'peer-a-public-key',
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
assert.deepEqual(normalized.peer_urls, ['tcp://peer-a:11010'])
|
||||
assert.deepEqual(normalized.peers, [
|
||||
{
|
||||
uri: 'tcp://peer-a:11010',
|
||||
peer_public_key: 'peer-a-public-key',
|
||||
},
|
||||
])
|
||||
|
||||
const unchangedUrl = toBackendNetworkConfig({
|
||||
...normalized,
|
||||
peer_urls: ['tcp://peer-a:11010', 'tcp://peer-b:11010'],
|
||||
})
|
||||
|
||||
assert.equal(unchangedUrl.peers[0].peer_public_key, 'peer-a-public-key')
|
||||
assert.deepEqual(unchangedUrl.peers[1], { uri: 'tcp://peer-b:11010' })
|
||||
|
||||
const changedUrl = toBackendNetworkConfig({
|
||||
...normalized,
|
||||
peer_urls: ['tcp://peer-c:11010'],
|
||||
})
|
||||
|
||||
assert.deepEqual(changedUrl.peers, [{ uri: 'tcp://peer-c:11010' }])
|
||||
|
||||
const clearedUrls = toBackendNetworkConfig({
|
||||
...normalized,
|
||||
peer_urls: [],
|
||||
})
|
||||
|
||||
assert.deepEqual(clearedUrls.peer_urls ?? [], [])
|
||||
assert.deepEqual(clearedUrls.peers ?? [], [])
|
||||
}
|
||||
|
||||
function assertNumberBoundaries() {
|
||||
@@ -530,7 +469,6 @@ const tests = [
|
||||
assertEnumCompatibility,
|
||||
assertAclDefaultsAndExplicitZero,
|
||||
assertNetworkingMethodNormalization,
|
||||
assertPeerPublicKeysPreserved,
|
||||
assertNumberBoundaries,
|
||||
]
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ const currentNetworkConfig = ref<NetworkTypes.NetworkConfig | undefined>(undefin
|
||||
const listInstanceIdResponse = ref<Api.ListNetworkInstanceIdResponse | undefined>(undefined);
|
||||
|
||||
const isRunning = (instanceId: string) => {
|
||||
return (listInstanceIdResponse.value?.running_inst_ids ?? []).map(Utils.UuidToStr).includes(instanceId);
|
||||
return listInstanceIdResponse.value?.running_inst_ids.map(Utils.UuidToStr).includes(instanceId);
|
||||
}
|
||||
|
||||
const networkMetaCache = ref<Record<string, Api.NetworkMeta>>({});
|
||||
@@ -46,7 +46,7 @@ const loadNetworkMetas = async (instanceIds: string[]) => {
|
||||
|
||||
try {
|
||||
const response = await props.api.get_network_metas(missingIds);
|
||||
Object.assign(networkMetaCache.value, response.metas ?? {});
|
||||
Object.assign(networkMetaCache.value, response.metas);
|
||||
} catch (e) {
|
||||
console.error("Failed to load network metas", e);
|
||||
}
|
||||
@@ -80,8 +80,8 @@ const updateInstanceList = () => {
|
||||
let insts = new Set<string>();
|
||||
let t = listInstanceIdResponse.value;
|
||||
if (t) {
|
||||
(t.running_inst_ids ?? []).forEach((u) => insts.add(Utils.UuidToStr(u)));
|
||||
(t.disabled_inst_ids ?? []).forEach((u) => insts.add(Utils.UuidToStr(u)));
|
||||
t.running_inst_ids.forEach((u) => insts.add(Utils.UuidToStr(u)));
|
||||
t.disabled_inst_ids.forEach((u) => insts.add(Utils.UuidToStr(u)));
|
||||
}
|
||||
|
||||
const newList = Array.from(insts).map((instance: string) => {
|
||||
@@ -149,7 +149,7 @@ const networkIsDisabled = computed(() => {
|
||||
if (!selectedInstanceId.value) {
|
||||
return false;
|
||||
}
|
||||
return (listInstanceIdResponse.value?.disabled_inst_ids ?? []).map(Utils.UuidToStr).includes(selectedInstanceId.value?.uuid);
|
||||
return listInstanceIdResponse.value?.disabled_inst_ids.map(Utils.UuidToStr).includes(selectedInstanceId.value?.uuid);
|
||||
});
|
||||
watch(networkIsDisabled, async (newVal, oldVal) => {
|
||||
if (newVal !== oldVal && newVal === true) {
|
||||
@@ -287,35 +287,17 @@ const loadNetworkInstanceIds = async () => {
|
||||
}
|
||||
|
||||
const loadCurrentNetworkInfo = async () => {
|
||||
const selected = selectedInstanceId.value?.uuid;
|
||||
if (!selected) {
|
||||
curNetworkInfo.value = null;
|
||||
if (!selectedInstanceId.value) {
|
||||
return;
|
||||
}
|
||||
if (!needShowNetworkStatus.value) {
|
||||
curNetworkInfo.value = null;
|
||||
return;
|
||||
}
|
||||
if (curNetworkInfo.value?.instance_id !== selected) {
|
||||
curNetworkInfo.value = null;
|
||||
}
|
||||
|
||||
let network_info = await props.api.get_network_info(selected);
|
||||
if (selectedInstanceId.value?.uuid !== selected) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!network_info) {
|
||||
curNetworkInfo.value = {
|
||||
instance_id: selected,
|
||||
running: false,
|
||||
error_msg: t('web.device_management.network_info_unavailable'),
|
||||
} as NetworkTypes.NetworkInstance;
|
||||
return;
|
||||
}
|
||||
let network_info = await props.api.get_network_info(selectedInstanceId.value.uuid);
|
||||
|
||||
curNetworkInfo.value = {
|
||||
instance_id: selected,
|
||||
instance_id: selectedInstanceId.value.uuid,
|
||||
running: network_info?.running ?? false,
|
||||
error_msg: network_info?.error_msg ?? '',
|
||||
detail: network_info,
|
||||
@@ -510,7 +492,7 @@ onUnmounted(() => {
|
||||
<div class="flex items-center min-w-0">
|
||||
<div class="mr-4 min-w-0 flex-1">
|
||||
<span class="truncate block">{{ t('network_name') }}: {{
|
||||
slotProps.option.meta?.network_name ?? slotProps.option.uuid }}</span>
|
||||
slotProps.option.meta.network_name }}</span>
|
||||
</div>
|
||||
<Tag class="my-auto leading-3 shrink-0"
|
||||
:severity="isRunning(slotProps.option.uuid) ? 'success' : 'info'"
|
||||
@@ -587,13 +569,10 @@ onUnmounted(() => {
|
||||
<h2 class="text-xl font-medium">{{ t('web.device_management.network_status') }}</h2>
|
||||
</div>
|
||||
|
||||
<Status v-if="curNetworkInfo && curNetworkInfo.error_msg === ''" v-bind:cur-network-inst="curNetworkInfo"
|
||||
<Status v-if="(curNetworkInfo?.error_msg ?? '') === ''" v-bind:cur-network-inst="curNetworkInfo"
|
||||
class="mb-4">
|
||||
</Status>
|
||||
<Message v-else-if="curNetworkInfo?.error_msg" severity="error" class="mb-4">{{
|
||||
curNetworkInfo.error_msg }}</Message>
|
||||
<Message v-else severity="info" class="mb-4">{{ t('web.device_management.loading_network_status') }}
|
||||
</Message>
|
||||
<Message v-else severity="error" class="mb-4">{{ curNetworkInfo?.error_msg }}</Message>
|
||||
|
||||
<div class="text-center mt-4">
|
||||
<Button @click="stopNetwork" :disabled="!currentNetworkControl.deletable.value"
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import { useTimeAgo } from '@vueuse/core'
|
||||
import { IPv4 } from 'ip-num/IPNumber'
|
||||
import { NetworkInstance, type TunnelInfo, type NodeInfo, type PeerRoutePair } from '../types/network'
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { computed, onMounted, onUnmounted, ref } from 'vue';
|
||||
import { ipv4InetToString, ipv4ToString, ipv6ToString } from '../modules/utils';
|
||||
import { latencyMs, lossRate, numericValue, peerConns } from '../modules/statusDisplay';
|
||||
import { Badge, DataTable, Column, Tag, Chip, Button, Dialog, ScrollPanel, Timeline, Divider, Card, } from 'primevue';
|
||||
import NetworkChart from './NetworkChart.vue';
|
||||
|
||||
@@ -39,8 +39,8 @@ function routeCost(info: any) {
|
||||
return '?'
|
||||
}
|
||||
|
||||
function resolveObjPath(path: string, obj: any = globalThis, separator = '.') {
|
||||
const properties = path.split(separator)
|
||||
function resolveObjPath(path: string, obj = globalThis, separator = '.') {
|
||||
const properties = Array.isArray(path) ? path : path.split(separator)
|
||||
return properties.reduce((prev, curr) => prev?.[curr], obj)
|
||||
}
|
||||
|
||||
@@ -48,17 +48,10 @@ function statsCommon(info: any, field: string): number | undefined {
|
||||
if (!info.peer)
|
||||
return undefined
|
||||
|
||||
let sum = 0
|
||||
let hasValue = false
|
||||
for (const conn of peerConns(info)) {
|
||||
const value = numericValue(resolveObjPath(field, conn))
|
||||
if (value === undefined)
|
||||
continue
|
||||
|
||||
sum += value
|
||||
hasValue = true
|
||||
}
|
||||
return hasValue ? sum : undefined
|
||||
const conns = info.peer.conns
|
||||
return conns.reduce((acc: number, conn: any) => {
|
||||
return acc + resolveObjPath(field, conn)
|
||||
}, 0)
|
||||
}
|
||||
|
||||
function humanFileSize(bytes: number, si = false, dp = 1) {
|
||||
@@ -81,6 +74,14 @@ function humanFileSize(bytes: number, si = false, dp = 1) {
|
||||
return `${bytes.toFixed(dp)} ${units[u]}`
|
||||
}
|
||||
|
||||
function latencyMs(info: PeerRoutePair) {
|
||||
let lat_us_sum = statsCommon(info, 'stats.latency_us')
|
||||
if (lat_us_sum === undefined)
|
||||
return ''
|
||||
lat_us_sum = lat_us_sum / 1000 / info.peer!.conns.length
|
||||
return `${lat_us_sum % 1 > 0 ? Math.round(lat_us_sum) + 1 : Math.round(lat_us_sum)}ms`
|
||||
}
|
||||
|
||||
function txBytes(info: PeerRoutePair) {
|
||||
const tx = statsCommon(info, 'stats.tx_bytes')
|
||||
return tx ? humanFileSize(tx) : ''
|
||||
@@ -91,6 +92,11 @@ function rxBytes(info: PeerRoutePair) {
|
||||
return rx ? humanFileSize(rx) : ''
|
||||
}
|
||||
|
||||
function lossRate(info: PeerRoutePair) {
|
||||
const lossRate = statsCommon(info, 'loss_rate')
|
||||
return lossRate !== undefined ? `${Math.round(lossRate * 100)}%` : ''
|
||||
}
|
||||
|
||||
function version(info: PeerRoutePair) {
|
||||
return info.route.version === '' ? 'unknown' : info.route.version
|
||||
}
|
||||
@@ -99,7 +105,7 @@ function ipFormat(info: PeerRoutePair) {
|
||||
const ip = info.route.ipv4_addr
|
||||
if (typeof ip === 'string')
|
||||
return ip
|
||||
return ip ? ipv4InetToString(ip) : ''
|
||||
return ip ? `${IPv4.fromNumber(ip.address.addr)}/${ip.network_length}` : ''
|
||||
}
|
||||
|
||||
function oneTunnelProto(tunnel?: TunnelInfo): string {
|
||||
@@ -125,7 +131,7 @@ function oneTunnelProto(tunnel?: TunnelInfo): string {
|
||||
}
|
||||
|
||||
function tunnelProto(info: PeerRoutePair) {
|
||||
return [...new Set(peerConns(info).map(c => oneTunnelProto(c.tunnel)))].join(',')
|
||||
return [...new Set(info.peer?.conns.map(c => oneTunnelProto(c.tunnel)))].join(',')
|
||||
}
|
||||
|
||||
const myNodeInfo = computed(() => {
|
||||
@@ -200,7 +206,7 @@ const myNodeInfoChips = computed(() => {
|
||||
|
||||
// local ipv4s
|
||||
const local_ipv4s = my_node_info.ips?.interface_ipv4s
|
||||
for (const [idx, ip] of local_ipv4s?.entries() ?? []) {
|
||||
for (const [idx, ip] of local_ipv4s?.entries()) {
|
||||
chips.push({
|
||||
label: `Local IPv4 ${idx}: ${ipv4ToString(ip)}`,
|
||||
icon: '',
|
||||
@@ -209,7 +215,7 @@ const myNodeInfoChips = computed(() => {
|
||||
|
||||
// local ipv6s
|
||||
const local_ipv6s = my_node_info.ips?.interface_ipv6s
|
||||
for (const [idx, ip] of local_ipv6s?.entries() ?? []) {
|
||||
for (const [idx, ip] of local_ipv6s?.entries()) {
|
||||
chips.push({
|
||||
label: `Local IPv6 ${idx}: ${ipv6ToString(ip)}`,
|
||||
icon: '',
|
||||
@@ -220,7 +226,7 @@ const myNodeInfoChips = computed(() => {
|
||||
const public_ip = my_node_info.ips?.public_ipv4
|
||||
if (public_ip) {
|
||||
chips.push({
|
||||
label: `Public IP: ${ipv4ToString(public_ip)}`,
|
||||
label: `Public IP: ${IPv4.fromNumber(public_ip.addr)}`,
|
||||
icon: '',
|
||||
} as Chip)
|
||||
}
|
||||
@@ -235,7 +241,7 @@ const myNodeInfoChips = computed(() => {
|
||||
|
||||
// listeners:
|
||||
const listeners = my_node_info.listeners
|
||||
for (const [idx, listener] of listeners?.entries() ?? []) {
|
||||
for (const [idx, listener] of listeners?.entries()) {
|
||||
chips.push({
|
||||
label: `Listener ${idx}: ${listener.url}`,
|
||||
icon: '',
|
||||
@@ -282,14 +288,6 @@ function natType(info: PeerRoutePair): string {
|
||||
return ''
|
||||
}
|
||||
|
||||
function isPublicServerRoute(info: PeerRoutePair): boolean {
|
||||
return info.route?.feature_flag?.is_public_server ?? false
|
||||
}
|
||||
|
||||
function shouldAvoidRelayData(info: PeerRoutePair): boolean {
|
||||
return info.route?.feature_flag?.avoid_relay_data ?? false
|
||||
}
|
||||
|
||||
const peerCount = computed(() => {
|
||||
if (!peerRouteInfos.value)
|
||||
return 0
|
||||
@@ -344,7 +342,7 @@ function showEventLogs() {
|
||||
if (!detail)
|
||||
return
|
||||
|
||||
dialogContent.value = detail.events?.map((event: string) => JSON.parse(event)) ?? []
|
||||
dialogContent.value = detail.events.map((event: string) => JSON.parse(event))
|
||||
dialogHeader.value = 'event_log'
|
||||
dialogVisible.value = true
|
||||
}
|
||||
@@ -436,16 +434,16 @@ function showEventLogs() {
|
||||
<Column :field="ipFormat" :header="t('virtual_ipv4')" />
|
||||
<Column :header="t('hostname')">
|
||||
<template #body="slotProps">
|
||||
<div v-if="!slotProps.data.route.cost || !isPublicServerRoute(slotProps.data)"
|
||||
<div v-if="!slotProps.data.route.cost || !slotProps.data.route.feature_flag.is_public_server"
|
||||
v-tooltip="slotProps.data.route.hostname">
|
||||
{{
|
||||
slotProps.data.route.hostname }}
|
||||
</div>
|
||||
<div v-else v-tooltip="slotProps.data.route.hostname" class="space-x-1">
|
||||
<Tag v-if="isPublicServerRoute(slotProps.data)" severity="info" value="Info">
|
||||
<Tag v-if="slotProps.data.route.feature_flag.is_public_server" severity="info" value="Info">
|
||||
{{ t('status.server') }}
|
||||
</Tag>
|
||||
<Tag v-if="shouldAvoidRelayData(slotProps.data)" severity="warn" value="Warn">
|
||||
<Tag v-if="slotProps.data.route.feature_flag.avoid_relay_data" severity="warn" value="Warn">
|
||||
{{ t('status.relay') }}
|
||||
</Tag>
|
||||
</div>
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { Button, Column, DataTable, Divider, InputText, Select, SelectButton, ToggleButton } from 'primevue'
|
||||
import { ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { AclAction, AclChain, AclChainType, AclProtocol, AclRule, ensureAclChain, ensureAclRuleLists } from '../../types/network'
|
||||
import { AclAction, AclChain, AclChainType, AclProtocol, AclRule } from '../../types/network'
|
||||
import AclRuleDialog from './AclRuleDialog.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
@@ -13,11 +13,7 @@ const chain = defineModel<AclChain>({ required: true })
|
||||
|
||||
const { t } = useI18n()
|
||||
|
||||
function rules() {
|
||||
return ensureAclChain(chain.value).rules
|
||||
}
|
||||
|
||||
watch(() => rules(), (newRules) => {
|
||||
watch(() => chain.value.rules, (newRules) => {
|
||||
if (!newRules) return
|
||||
const isSorted = newRules.every((rule, i) => i === 0 || (rule.priority || 0) <= (newRules[i - 1].priority || 0))
|
||||
if (!isSorted) {
|
||||
@@ -64,7 +60,7 @@ function addRule() {
|
||||
editingRule.value = {
|
||||
name: '',
|
||||
description: '',
|
||||
priority: rules().length,
|
||||
priority: chain.value.rules.length,
|
||||
enabled: true,
|
||||
protocol: AclProtocol.Any,
|
||||
ports: [],
|
||||
@@ -83,31 +79,28 @@ function addRule() {
|
||||
|
||||
function editRule(index: number) {
|
||||
editingRuleIndex.value = index
|
||||
editingRule.value = ensureAclRuleLists(JSON.parse(JSON.stringify(rules()[index])))
|
||||
editingRule.value = JSON.parse(JSON.stringify(chain.value.rules[index]))
|
||||
showRuleDialog.value = true
|
||||
}
|
||||
|
||||
function deleteRule(index: number) {
|
||||
rules().splice(index, 1)
|
||||
chain.value.rules.splice(index, 1)
|
||||
}
|
||||
|
||||
function saveRule(rule: AclRule) {
|
||||
const chainRules = rules()
|
||||
ensureAclRuleLists(rule)
|
||||
if (editingRuleIndex.value === -1) {
|
||||
chainRules.push(rule)
|
||||
chain.value.rules.push(rule)
|
||||
} else {
|
||||
chainRules[editingRuleIndex.value] = rule
|
||||
chain.value.rules[editingRuleIndex.value] = rule
|
||||
}
|
||||
chainRules.sort((a, b) => (b.priority || 0) - (a.priority || 0))
|
||||
chain.value.rules.sort((a, b) => (b.priority || 0) - (a.priority || 0))
|
||||
}
|
||||
|
||||
function onRowReorder(event: any) {
|
||||
chain.value.rules = event.value ?? []
|
||||
const chainRules = rules()
|
||||
chain.value.rules = event.value
|
||||
// Update priorities based on new order (higher priority at top)
|
||||
chainRules.forEach((rule, index) => {
|
||||
rule.priority = chainRules.length - index - 1
|
||||
chain.value.rules.forEach((rule, index) => {
|
||||
rule.priority = chain.value.rules.length - index - 1
|
||||
})
|
||||
}
|
||||
</script>
|
||||
@@ -150,7 +143,7 @@ function onRowReorder(event: any) {
|
||||
<Button icon="pi pi-plus" :label="t('acl.add_rule')" severity="success" size="small" @click="addRule" />
|
||||
</div>
|
||||
|
||||
<DataTable :value="rules()" @row-reorder="onRowReorder" responsiveLayout="scroll">
|
||||
<DataTable :value="chain.rules" @row-reorder="onRowReorder" responsiveLayout="scroll">
|
||||
<Column rowReorder headerStyle="width: 3rem" />
|
||||
<Column field="enabled" :header="t('acl.rule.enabled')">
|
||||
<template #body="{ data }">
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<script setup lang="ts">
|
||||
import { Button, Column, DataTable, Dialog, InputText, MultiSelect, Password } from 'primevue';
|
||||
import { computed, ref } from 'vue';
|
||||
import { ref } from 'vue';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { GroupIdentity, GroupInfo, ensureGroupInfo } from '../../types/network';
|
||||
import { GroupIdentity, GroupInfo } from '../../types/network';
|
||||
|
||||
const props = defineProps<{
|
||||
groupNames?: string[]
|
||||
@@ -18,17 +18,6 @@ const editingGroupIndex = ref(-1)
|
||||
const showGroupDialog = ref(false)
|
||||
const oldGroupName = ref('')
|
||||
|
||||
function groupInfo() {
|
||||
return ensureGroupInfo(group.value)
|
||||
}
|
||||
|
||||
const members = computed({
|
||||
get: () => groupInfo().members,
|
||||
set: value => {
|
||||
groupInfo().members = value
|
||||
},
|
||||
})
|
||||
|
||||
function addGroup() {
|
||||
editingGroupIndex.value = -1
|
||||
editingGroup.value = {
|
||||
@@ -41,13 +30,13 @@ function addGroup() {
|
||||
|
||||
function editGroup(index: number) {
|
||||
editingGroupIndex.value = index
|
||||
editingGroup.value = JSON.parse(JSON.stringify(groupInfo().declares[index]))
|
||||
editingGroup.value = JSON.parse(JSON.stringify(group.value.declares[index]))
|
||||
oldGroupName.value = editingGroup.value?.group_name || ''
|
||||
showGroupDialog.value = true
|
||||
}
|
||||
|
||||
function deleteGroup(index: number) {
|
||||
groupInfo().declares.splice(index, 1)
|
||||
group.value.declares.splice(index, 1)
|
||||
}
|
||||
|
||||
function saveGroup() {
|
||||
@@ -55,15 +44,15 @@ function saveGroup() {
|
||||
const newName = editingGroup.value.group_name
|
||||
|
||||
if (editingGroupIndex.value === -1) {
|
||||
groupInfo().declares.push(editingGroup.value)
|
||||
group.value.declares.push(editingGroup.value)
|
||||
} else {
|
||||
if (oldGroupName.value && oldGroupName.value !== newName) {
|
||||
// Sync in members
|
||||
groupInfo().members = groupInfo().members.map(m => m === oldGroupName.value ? newName : m)
|
||||
group.value.members = group.value.members.map(m => m === oldGroupName.value ? newName : m)
|
||||
// Notify parent to sync in rules
|
||||
emit('rename-group', { oldName: oldGroupName.value, newName })
|
||||
}
|
||||
groupInfo().declares[editingGroupIndex.value] = editingGroup.value
|
||||
group.value.declares[editingGroupIndex.value] = editingGroup.value
|
||||
}
|
||||
showGroupDialog.value = false
|
||||
}
|
||||
@@ -81,7 +70,7 @@ function saveGroup() {
|
||||
<Button icon="pi pi-plus" :label="t('web.common.add')" severity="success" @click="addGroup" />
|
||||
</div>
|
||||
|
||||
<DataTable :value="groupInfo().declares" responsiveLayout="scroll">
|
||||
<DataTable :value="group.declares" responsiveLayout="scroll">
|
||||
<Column field="group_name" :header="t('acl.group.name')" />
|
||||
<Column field="group_secret" :header="t('acl.group.secret')">
|
||||
<template #body="{ data }">
|
||||
@@ -101,7 +90,7 @@ function saveGroup() {
|
||||
|
||||
<div class="flex flex-col gap-2">
|
||||
<label class="font-bold text-lg">{{ t('acl.group.members') }}</label>
|
||||
<MultiSelect v-model="members" :options="props.groupNames" multiple fluid filter
|
||||
<MultiSelect v-model="group.members" :options="props.groupNames" multiple fluid filter
|
||||
:placeholder="t('acl.group.members')" />
|
||||
</div>
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { Button, Menu, Tab, TabList, TabPanel, TabPanels, Tabs } from 'primevue'
|
||||
import { computed, ref } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { Acl, AclAction, AclChainType, ensureAclV1 } from '../../types/network'
|
||||
import { Acl, AclAction, AclChainType } from '../../types/network'
|
||||
import AclChainEditor from './AclChainEditor.vue'
|
||||
import AclGroupEditor from './AclGroupEditor.vue'
|
||||
|
||||
@@ -12,7 +12,6 @@ const { t } = useI18n()
|
||||
|
||||
const activeTab = ref(0)
|
||||
const menu = ref()
|
||||
const aclV1 = computed(() => ensureAclV1(acl.value))
|
||||
|
||||
const addMenuModel = ref([
|
||||
{ label: () => t('acl.inbound'), command: () => addChain(AclChainType.Inbound) },
|
||||
@@ -21,6 +20,10 @@ const addMenuModel = ref([
|
||||
])
|
||||
|
||||
function addChain(type: AclChainType) {
|
||||
if (!acl.value.acl_v1) {
|
||||
acl.value.acl_v1 = { chains: [], group: { declares: [], members: [] } }
|
||||
}
|
||||
|
||||
let defaultName = ''
|
||||
switch (type) {
|
||||
case AclChainType.Inbound: defaultName = 'Inbound'; break;
|
||||
@@ -28,7 +31,7 @@ function addChain(type: AclChainType) {
|
||||
case AclChainType.Forward: defaultName = 'Forward'; break;
|
||||
}
|
||||
|
||||
aclV1.value.chains.push({
|
||||
acl.value.acl_v1.chains.push({
|
||||
name: defaultName,
|
||||
chain_type: type,
|
||||
description: '',
|
||||
@@ -37,20 +40,21 @@ function addChain(type: AclChainType) {
|
||||
default_action: AclAction.Allow
|
||||
})
|
||||
|
||||
activeTab.value = aclV1.value.chains.length - 1
|
||||
activeTab.value = acl.value.acl_v1.chains.length - 1
|
||||
}
|
||||
|
||||
function removeChain(index: number) {
|
||||
if (confirm(t('acl.delete_chain_confirm'))) {
|
||||
aclV1.value.chains.splice(index, 1)
|
||||
if (activeTab.value >= aclV1.value.chains.length) {
|
||||
activeTab.value = Math.max(0, aclV1.value.chains.length)
|
||||
acl.value.acl_v1?.chains.splice(index, 1)
|
||||
if (activeTab.value >= (acl.value.acl_v1?.chains.length || 0)) {
|
||||
activeTab.value = Math.max(0, (acl.value.acl_v1?.chains.length || 0))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function handleRenameGroup({ oldName, newName }: { oldName: string, newName: string }) {
|
||||
aclV1.value.chains.forEach(chain => {
|
||||
if (!acl.value.acl_v1) return
|
||||
acl.value.acl_v1.chains.forEach(chain => {
|
||||
chain.rules.forEach(rule => {
|
||||
rule.source_groups = rule.source_groups.map(g => g === oldName ? newName : g)
|
||||
rule.destination_groups = rule.destination_groups.map(g => g === oldName ? newName : g)
|
||||
@@ -59,11 +63,11 @@ function handleRenameGroup({ oldName, newName }: { oldName: string, newName: str
|
||||
}
|
||||
|
||||
const groupNames = computed(() => {
|
||||
return aclV1.value.group?.declares.map(g => g.group_name) || []
|
||||
return acl.value.acl_v1?.group?.declares.map(g => g.group_name) || []
|
||||
})
|
||||
|
||||
const tabs = computed(() => {
|
||||
const chains = aclV1.value.chains
|
||||
const chains = acl.value.acl_v1?.chains || []
|
||||
const result: { type: string, label: string, index: number }[] = []
|
||||
|
||||
if (chains.length === 0) {
|
||||
@@ -120,13 +124,24 @@ const tabs = computed(() => {
|
||||
</div>
|
||||
|
||||
<!-- Rule Chains -->
|
||||
<div v-if="tab.type === 'chain' && aclV1.chains[tab.index]" class="py-4">
|
||||
<AclChainEditor v-model="aclV1.chains[tab.index]" :group-names="groupNames" />
|
||||
<div v-if="tab.type === 'chain' && acl.acl_v1 && acl.acl_v1.chains[tab.index]" class="py-4">
|
||||
<AclChainEditor v-model="acl.acl_v1.chains[tab.index]" :group-names="groupNames" />
|
||||
</div>
|
||||
|
||||
<!-- Group Management -->
|
||||
<div v-if="tab.type === 'groups'" class="py-4">
|
||||
<AclGroupEditor v-model="aclV1.group" :group-names="groupNames" @rename-group="handleRenameGroup" />
|
||||
<template v-if="acl.acl_v1">
|
||||
<AclGroupEditor v-if="acl.acl_v1.group" v-model="acl.acl_v1.group" :group-names="groupNames"
|
||||
@rename-group="handleRenameGroup" />
|
||||
<div v-else class="flex justify-center p-4">
|
||||
<Button :label="t('web.common.add') + ' ' + t('acl.groups')"
|
||||
@click="acl.acl_v1.group = { declares: [], members: [] }" />
|
||||
</div>
|
||||
</template>
|
||||
<div v-else class="flex justify-center p-4">
|
||||
<Button :label="t('acl.enabled')"
|
||||
@click="acl.acl_v1 = { chains: [], group: { declares: [], members: [] } }" />
|
||||
</div>
|
||||
</div>
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<script setup lang="ts">
|
||||
import { AutoComplete, Button, Checkbox, Dialog, InputNumber, InputText, MultiSelect, Panel, SelectButton, ToggleButton } from 'primevue';
|
||||
import { computed, ref, watch } from 'vue';
|
||||
import { computed, ref } from 'vue';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { AclAction, AclProtocol, AclRule, ensureAclRuleLists } from '../../types/network';
|
||||
import { AclAction, AclProtocol, AclRule } from '../../types/network';
|
||||
|
||||
const props = defineProps<{
|
||||
visible: boolean
|
||||
@@ -32,8 +32,6 @@ const showPorts = computed(() => {
|
||||
return rule.value.protocol === AclProtocol.TCP || rule.value.protocol === AclProtocol.UDP || rule.value.protocol === AclProtocol.Any
|
||||
})
|
||||
|
||||
watch(() => rule.value, ensureAclRuleLists, { immediate: true })
|
||||
|
||||
function close() {
|
||||
emit('update:visible', false)
|
||||
}
|
||||
|
||||
@@ -341,8 +341,6 @@ web:
|
||||
import_config: 导入配置
|
||||
create_new: 创建新网络
|
||||
network_status: 网络状态
|
||||
loading_network_status: 正在加载网络状态
|
||||
network_info_unavailable: 网络状态不可用
|
||||
network_configuration: 网络配置
|
||||
loading_network_configuration: 加载网络配置
|
||||
no_network_selected: 未选择网络
|
||||
|
||||
@@ -341,8 +341,6 @@ web:
|
||||
import_config: Import Config
|
||||
create_new: Create New Network
|
||||
network_status: Network Status
|
||||
loading_network_status: Loading Network Status
|
||||
network_info_unavailable: Network status is unavailable
|
||||
network_configuration: Network Configuration
|
||||
loading_network_configuration: Loading Network Configuration
|
||||
no_network_selected: No Network Selected
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
import type { PeerRoutePair } from '../types/network'
|
||||
|
||||
export function numericValue(value: unknown): number | undefined {
|
||||
if (typeof value === 'number')
|
||||
return Number.isFinite(value) ? value : undefined
|
||||
|
||||
if (typeof value !== 'string' || value.trim() === '')
|
||||
return undefined
|
||||
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) ? parsed : undefined
|
||||
}
|
||||
|
||||
export function peerConns(info: PeerRoutePair) {
|
||||
return info.peer?.conns || []
|
||||
}
|
||||
|
||||
function defaultConnId(info: PeerRoutePair) {
|
||||
const defaultConn = info.peer?.default_conn_id
|
||||
if (!defaultConn)
|
||||
return undefined
|
||||
|
||||
const part1 = defaultConn.part1 ?? 0
|
||||
const part2 = defaultConn.part2 ?? 0
|
||||
const part3 = defaultConn.part3 ?? 0
|
||||
const part4 = defaultConn.part4 ?? 0
|
||||
if (part1 === 0 && part2 === 0 && part3 === 0 && part4 === 0)
|
||||
return undefined
|
||||
|
||||
const toHex = (value: number) => value.toString(16).padStart(8, '0')
|
||||
const part1Hex = toHex(part1)
|
||||
const part2Hex = toHex(part2)
|
||||
const part3Hex = toHex(part3)
|
||||
const part4Hex = toHex(part4)
|
||||
return `${part1Hex}-${part2Hex.slice(0, 4)}-${part2Hex.slice(4, 8)}-${part3Hex.slice(0, 4)}-${part3Hex.slice(4, 8)}${part4Hex}`
|
||||
}
|
||||
|
||||
function defaultConnFirst(info: PeerRoutePair) {
|
||||
const conns = peerConns(info)
|
||||
const connId = defaultConnId(info)
|
||||
if (!connId)
|
||||
return conns
|
||||
|
||||
const defaultConn = conns.find(conn => conn.conn_id === connId)
|
||||
return defaultConn ? [defaultConn, ...conns.filter(conn => conn !== defaultConn)] : conns
|
||||
}
|
||||
|
||||
export function latencyMs(info: PeerRoutePair) {
|
||||
const connId = defaultConnId(info)
|
||||
let minLatencyUs: number | undefined
|
||||
|
||||
for (const conn of peerConns(info)) {
|
||||
if (!conn.stats)
|
||||
continue
|
||||
|
||||
const latencyUs = numericValue(conn.stats.latency_us)
|
||||
if (latencyUs === undefined)
|
||||
continue
|
||||
|
||||
if (connId === conn.conn_id)
|
||||
return `${Math.ceil(latencyUs / 1000)}ms`
|
||||
|
||||
minLatencyUs = Math.min(minLatencyUs ?? latencyUs, latencyUs)
|
||||
}
|
||||
|
||||
if (minLatencyUs === undefined)
|
||||
return ''
|
||||
|
||||
return `${Math.ceil(minLatencyUs / 1000)}ms`
|
||||
}
|
||||
|
||||
export function lossRate(info: PeerRoutePair) {
|
||||
for (const conn of defaultConnFirst(info)) {
|
||||
const loss = numericValue(conn.loss_rate)
|
||||
if (loss === undefined)
|
||||
continue
|
||||
|
||||
return `${Math.round(loss * 100)}%`
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
@@ -1,30 +1,24 @@
|
||||
import { IPv4, IPv6 } from 'ip-num/IPNumber'
|
||||
import { Ipv4Addr, Ipv4Inet, Ipv6Addr } from '../types/network'
|
||||
|
||||
export function ipv4ToString(ip: Ipv4Addr | null | undefined) {
|
||||
if (!ip) {
|
||||
return ''
|
||||
}
|
||||
return IPv4.fromNumber(ip.addr ?? 0).toString()
|
||||
export function ipv4ToString(ip: Ipv4Addr) {
|
||||
return IPv4.fromNumber(ip.addr).toString()
|
||||
}
|
||||
|
||||
export function ipv4InetToString(ip: Ipv4Inet | undefined) {
|
||||
if (ip?.address === undefined) {
|
||||
return 'undefined'
|
||||
}
|
||||
return `${ipv4ToString(ip.address)}/${ip.network_length ?? 0}`
|
||||
return `${ipv4ToString(ip.address)}/${ip.network_length}`
|
||||
}
|
||||
|
||||
export function ipv6ToString(ip: Ipv6Addr | null | undefined) {
|
||||
if (!ip) {
|
||||
return ''
|
||||
}
|
||||
export function ipv6ToString(ip: Ipv6Addr) {
|
||||
return IPv6.fromBigInt(
|
||||
(BigInt(ip.part1 ?? 0) << BigInt(96))
|
||||
+ (BigInt(ip.part2 ?? 0) << BigInt(64))
|
||||
+ (BigInt(ip.part3 ?? 0) << BigInt(32))
|
||||
+ BigInt(ip.part4 ?? 0),
|
||||
).toString()
|
||||
(BigInt(ip.part1) << BigInt(96))
|
||||
+ (BigInt(ip.part2) << BigInt(64))
|
||||
+ (BigInt(ip.part3) << BigInt(32))
|
||||
+ BigInt(ip.part4),
|
||||
)
|
||||
}
|
||||
|
||||
function toHexString(uint64: bigint, padding = 9): string {
|
||||
@@ -49,17 +43,14 @@ function uint32ToUuid(part1: number, part2: number, part3: number, part4: number
|
||||
}
|
||||
|
||||
export interface UUID {
|
||||
part1?: number;
|
||||
part2?: number;
|
||||
part3?: number;
|
||||
part4?: number;
|
||||
part1: number;
|
||||
part2: number;
|
||||
part3: number;
|
||||
part4: number;
|
||||
}
|
||||
|
||||
export function UuidToStr(uuid: UUID | null | undefined): string {
|
||||
if (!uuid) {
|
||||
return '';
|
||||
}
|
||||
return uint32ToUuid(uuid.part1 ?? 0, uuid.part2 ?? 0, uuid.part3 ?? 0, uuid.part4 ?? 0);
|
||||
export function UuidToStr(uuid: UUID): string {
|
||||
return uint32ToUuid(uuid.part1, uuid.part2, uuid.part3, uuid.part4);
|
||||
}
|
||||
|
||||
export interface Location {
|
||||
@@ -80,12 +71,11 @@ export interface DeviceInfo {
|
||||
}
|
||||
|
||||
export function buildDeviceInfo(device: any): DeviceInfo {
|
||||
const runningInstances = device.info?.running_network_instances ?? [];
|
||||
let dev_info: DeviceInfo = {
|
||||
hostname: device.info?.hostname,
|
||||
public_ip: device.client_url,
|
||||
running_network_instances: runningInstances.map((instance: any) => UuidToStr(instance)),
|
||||
running_network_count: runningInstances.length,
|
||||
running_network_instances: device.info?.running_network_instances.map((instance: any) => UuidToStr(instance)),
|
||||
running_network_count: device.info?.running_network_instances.length,
|
||||
report_time: device.info?.report_time,
|
||||
easytier_version: device.info?.easytier_version,
|
||||
machine_id: UuidToStr(device.info?.machine_id),
|
||||
|
||||
@@ -2,7 +2,6 @@ import { v4 as uuidv4 } from 'uuid'
|
||||
import {
|
||||
NetworkConfig as NetworkConfigPb,
|
||||
NetworkingMethod,
|
||||
type NetworkPeerConfig,
|
||||
type NetworkConfig as ProtoNetworkConfig,
|
||||
type PortForwardConfig,
|
||||
} from '../generated/proto/api_manage'
|
||||
@@ -17,16 +16,11 @@ import {
|
||||
type GroupInfo,
|
||||
type Rule as AclRule,
|
||||
} from '../generated/proto/acl'
|
||||
import {
|
||||
CompressionAlgoPb,
|
||||
NatType,
|
||||
type PeerFeatureFlag,
|
||||
type SecureModeConfig,
|
||||
} from '../generated/proto/common'
|
||||
import { CompressionAlgoPb, NatType, type SecureModeConfig } from '../generated/proto/common'
|
||||
import { prepareNetworkConfigForProtoJson } from './networkCompat'
|
||||
|
||||
export { AclAction, AclChainType, AclProtocol, CompressionAlgoPb, NatType, NetworkingMethod }
|
||||
export type { Acl, AclChain, AclRule, AclV1, GroupIdentity, GroupInfo, NetworkPeerConfig, PeerFeatureFlag, PortForwardConfig, SecureModeConfig }
|
||||
export type { Acl, AclChain, AclRule, AclV1, GroupIdentity, GroupInfo, PortForwardConfig, SecureModeConfig }
|
||||
|
||||
export type NetworkConfig = Omit<
|
||||
ProtoNetworkConfig,
|
||||
@@ -38,39 +32,14 @@ export type NetworkConfig = Omit<
|
||||
networking_method: NetworkingMethod | string
|
||||
}
|
||||
|
||||
export type NormalizedAclV1 = AclV1 & {
|
||||
group: GroupInfo
|
||||
}
|
||||
|
||||
const UINT64_MAX = (1n << 64n) - 1n
|
||||
|
||||
interface NetworkingConfigFields {
|
||||
peer_urls: string[]
|
||||
peers?: NetworkPeerConfig[]
|
||||
public_server_url?: string
|
||||
networking_method?: NetworkingMethod | string
|
||||
}
|
||||
|
||||
interface NetworkingMethodOptions {
|
||||
fillPeerUrlsFromPeers?: boolean
|
||||
}
|
||||
|
||||
function emptyGroupInfo(): GroupInfo {
|
||||
return {
|
||||
declares: [],
|
||||
members: [],
|
||||
}
|
||||
}
|
||||
|
||||
function emptyAcl(): Acl {
|
||||
return {
|
||||
acl_v1: {
|
||||
group: emptyGroupInfo(),
|
||||
chains: [],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function DEFAULT_NETWORK_CONFIG(): NetworkConfig {
|
||||
return {
|
||||
...NetworkConfigPb.create(),
|
||||
@@ -141,7 +110,15 @@ export function DEFAULT_NETWORK_CONFIG(): NetworkConfig {
|
||||
enable_magic_dns: false,
|
||||
enable_private_mode: false,
|
||||
port_forwards: [],
|
||||
acl: emptyAcl(),
|
||||
acl: {
|
||||
acl_v1: {
|
||||
group: {
|
||||
declares: [],
|
||||
members: [],
|
||||
},
|
||||
chains: [],
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,88 +126,6 @@ function cleanPeerUrls(urls: string[] | undefined): string[] {
|
||||
return (urls ?? []).map((url) => url.trim()).filter((url) => url.length > 0)
|
||||
}
|
||||
|
||||
function cleanNetworkPeers(peers: NetworkPeerConfig[] | undefined): NetworkPeerConfig[] {
|
||||
return (peers ?? [])
|
||||
.map((peer) => ({
|
||||
...peer,
|
||||
uri: peer.uri.trim(),
|
||||
}))
|
||||
.filter((peer) => peer.uri.length > 0)
|
||||
}
|
||||
|
||||
function peersFromUrls(urls: string[], existingPeers: NetworkPeerConfig[]): NetworkPeerConfig[] {
|
||||
const peersByUri = new Map<string, NetworkPeerConfig>()
|
||||
for (const peer of existingPeers) {
|
||||
if (!peersByUri.has(peer.uri)) {
|
||||
peersByUri.set(peer.uri, peer)
|
||||
}
|
||||
}
|
||||
|
||||
return urls.map((uri) => ({
|
||||
...(peersByUri.get(uri) ?? {}),
|
||||
uri,
|
||||
}))
|
||||
}
|
||||
|
||||
export function ensureAclRuleLists(rule: AclRule): AclRule {
|
||||
rule.ports ??= []
|
||||
rule.source_ips ??= []
|
||||
rule.destination_ips ??= []
|
||||
rule.source_ports ??= []
|
||||
rule.source_groups ??= []
|
||||
rule.destination_groups ??= []
|
||||
return rule
|
||||
}
|
||||
|
||||
export function ensureAclChain(chain: AclChain): AclChain {
|
||||
chain.rules ??= []
|
||||
chain.rules.forEach(ensureAclRuleLists)
|
||||
return chain
|
||||
}
|
||||
|
||||
export function ensureGroupInfo(group: GroupInfo): GroupInfo {
|
||||
group.declares ??= []
|
||||
group.members ??= []
|
||||
return group
|
||||
}
|
||||
|
||||
export function ensureAclV1(acl: Acl): NormalizedAclV1 {
|
||||
acl.acl_v1 ??= { chains: [], group: emptyGroupInfo() }
|
||||
acl.acl_v1.chains ??= []
|
||||
acl.acl_v1.chains.forEach(ensureAclChain)
|
||||
acl.acl_v1.group = ensureGroupInfo(acl.acl_v1.group ?? emptyGroupInfo())
|
||||
return acl.acl_v1 as NormalizedAclV1
|
||||
}
|
||||
|
||||
function normalizeAcl(acl: Acl | undefined): Acl {
|
||||
const source = acl ?? emptyAcl()
|
||||
const aclV1 = source.acl_v1 ?? { chains: [], group: emptyGroupInfo() }
|
||||
return {
|
||||
...source,
|
||||
acl_v1: {
|
||||
...aclV1,
|
||||
chains: (aclV1.chains ?? []).map((chain) => ({
|
||||
...chain,
|
||||
rules: (chain.rules ?? []).map((rule) => ({ ...ensureAclRuleLists({ ...rule }) })),
|
||||
})),
|
||||
group: ensureGroupInfo({
|
||||
...(aclV1.group ?? emptyGroupInfo()),
|
||||
declares: aclV1.group?.declares ?? [],
|
||||
members: aclV1.group?.members ?? [],
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function isGroupInfoEmpty(group: GroupInfo | undefined): boolean {
|
||||
return (group?.declares?.length ?? 0) === 0 && (group?.members?.length ?? 0) === 0
|
||||
}
|
||||
|
||||
function isAclEmpty(acl: Acl | undefined): boolean {
|
||||
const aclV1 = acl?.acl_v1
|
||||
return !aclV1 || ((aclV1.chains?.length ?? 0) === 0 && isGroupInfoEmpty(aclV1.group))
|
||||
}
|
||||
|
||||
function normalizeUint64ForInput(v: bigint | number | string | null | undefined): number | string | null {
|
||||
if (v == null) return null
|
||||
|
||||
@@ -259,24 +154,15 @@ function toBackendUint64(v: number | bigint | string | null | undefined): bigint
|
||||
}
|
||||
}
|
||||
|
||||
function applyNetworkingMethod(
|
||||
config: NetworkingConfigFields,
|
||||
options: NetworkingMethodOptions = {},
|
||||
): void {
|
||||
const existingPeers = cleanNetworkPeers(config.peers)
|
||||
function applyNetworkingMethod(config: NetworkingConfigFields): void {
|
||||
config.peer_urls = cleanPeerUrls(config.peer_urls)
|
||||
if (options.fillPeerUrlsFromPeers && config.peer_urls.length === 0 && existingPeers.length > 0) {
|
||||
config.peer_urls = existingPeers.map((peer) => peer.uri)
|
||||
}
|
||||
|
||||
const publicServerUrl = config.public_server_url?.trim() ?? ''
|
||||
const networkingMethod = config.networking_method ?? NetworkingMethod.Manual
|
||||
|
||||
switch (networkingMethod) {
|
||||
case NetworkingMethod.PublicServer:
|
||||
config.peer_urls = publicServerUrl
|
||||
? [publicServerUrl]
|
||||
: (options.fillPeerUrlsFromPeers ? existingPeers.map((peer) => peer.uri) : [])
|
||||
config.peer_urls = publicServerUrl ? [publicServerUrl] : []
|
||||
break
|
||||
case NetworkingMethod.Manual:
|
||||
break
|
||||
@@ -288,7 +174,6 @@ function applyNetworkingMethod(
|
||||
|
||||
config.networking_method = NetworkingMethod.Manual
|
||||
config.public_server_url = ''
|
||||
config.peers = peersFromUrls(config.peer_urls, existingPeers)
|
||||
}
|
||||
|
||||
export function normalizeNetworkConfig(config: NetworkConfig): NetworkConfig {
|
||||
@@ -296,19 +181,11 @@ export function normalizeNetworkConfig(config: NetworkConfig): NetworkConfig {
|
||||
ignoreUnknownFields: true,
|
||||
}) as unknown as NetworkConfig
|
||||
|
||||
applyNetworkingMethod(normalized, { fillPeerUrlsFromPeers: true })
|
||||
applyNetworkingMethod(normalized)
|
||||
normalized.mtu = normalizeNumberForInput(normalized.mtu)
|
||||
normalized.instance_recv_bps_limit = normalizeUint64ForInput(
|
||||
normalized.instance_recv_bps_limit as any,
|
||||
)
|
||||
normalized.proxy_cidrs ??= []
|
||||
normalized.listener_urls ??= []
|
||||
normalized.relay_network_whitelist ??= []
|
||||
normalized.routes ??= []
|
||||
normalized.exit_nodes ??= []
|
||||
normalized.mapped_listeners ??= []
|
||||
normalized.port_forwards ??= []
|
||||
normalized.acl = config.acl === undefined ? undefined : normalizeAcl(normalized.acl)
|
||||
|
||||
return normalized
|
||||
}
|
||||
@@ -321,9 +198,6 @@ export function toBackendNetworkConfig(config: NetworkConfig): NetworkConfig {
|
||||
applyNetworkingMethod(backend)
|
||||
backend.mtu = normalizeNumberForInput(config.mtu) ?? undefined
|
||||
backend.instance_recv_bps_limit = toBackendUint64(config.instance_recv_bps_limit)
|
||||
if (config.acl === undefined || isAclEmpty(config.acl)) {
|
||||
backend.acl = undefined
|
||||
}
|
||||
|
||||
return NetworkConfigPb.toJson(backend, {
|
||||
useProtoFieldName: true,
|
||||
@@ -412,7 +286,6 @@ export interface Route {
|
||||
proxy_cidrs: string[]
|
||||
hostname: string
|
||||
stun_info?: StunInfo
|
||||
feature_flag?: PeerFeatureFlag
|
||||
inst_id: string
|
||||
version: string
|
||||
}
|
||||
@@ -420,7 +293,6 @@ export interface Route {
|
||||
export interface PeerInfo {
|
||||
peer_id: number
|
||||
conns: PeerConnInfo[]
|
||||
default_conn_id?: CommonUuid
|
||||
}
|
||||
|
||||
export interface PeerConnInfo {
|
||||
@@ -431,7 +303,7 @@ export interface PeerConnInfo {
|
||||
features: string[]
|
||||
tunnel?: TunnelInfo
|
||||
stats?: PeerConnStats
|
||||
loss_rate?: number | string
|
||||
loss_rate: number
|
||||
}
|
||||
|
||||
export interface PeerRoutePair {
|
||||
@@ -450,18 +322,11 @@ export interface TunnelInfo {
|
||||
}
|
||||
|
||||
export interface PeerConnStats {
|
||||
rx_bytes: number | string
|
||||
tx_bytes: number | string
|
||||
rx_packets: number | string
|
||||
tx_packets: number | string
|
||||
latency_us: number | string
|
||||
}
|
||||
|
||||
export interface CommonUuid {
|
||||
part1?: number
|
||||
part2?: number
|
||||
part3?: number
|
||||
part4?: number
|
||||
rx_bytes: number
|
||||
tx_bytes: number
|
||||
rx_packets: number
|
||||
tx_packets: number
|
||||
latency_us: number
|
||||
}
|
||||
|
||||
// 添加新行
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { latencyMs, lossRate } from '../src/modules/statusDisplay'
|
||||
import { ipv4ToString, ipv6ToString } from '../src/modules/utils'
|
||||
|
||||
function peerRoutePair(conns: any[]) {
|
||||
return {
|
||||
route: {
|
||||
ipv4_addr: '10.0.0.2',
|
||||
hostname: 'peer',
|
||||
version: 'test',
|
||||
},
|
||||
peer: {
|
||||
conns,
|
||||
},
|
||||
} as any
|
||||
}
|
||||
|
||||
function peerRoutePairWithDefaultConn(conns: any[], defaultConnId: string) {
|
||||
const [part1, part2, part3, part4] = defaultConnId
|
||||
.replaceAll('-', '')
|
||||
.match(/.{8}/g)!
|
||||
.map((part) => Number.parseInt(part, 16))
|
||||
|
||||
return {
|
||||
...peerRoutePair(conns),
|
||||
peer: {
|
||||
default_conn_id: {
|
||||
part1,
|
||||
part2,
|
||||
part3,
|
||||
part4,
|
||||
},
|
||||
conns,
|
||||
},
|
||||
} as any
|
||||
}
|
||||
|
||||
describe('status display helpers', () => {
|
||||
it('does not render missing IP values as zero addresses', () => {
|
||||
expect(ipv4ToString(undefined)).toBe('')
|
||||
expect(ipv4ToString(null)).toBe('')
|
||||
expect(ipv4ToString({} as any)).toBe('0.0.0.0')
|
||||
expect(ipv4ToString({ addr: 0 })).toBe('0.0.0.0')
|
||||
|
||||
expect(ipv6ToString(undefined)).toBe('')
|
||||
expect(ipv6ToString(null)).toBe('')
|
||||
expect(ipv6ToString({} as any)).toBe('::0')
|
||||
expect(ipv6ToString({ part1: 0, part2: 0, part3: 0, part4: 0 })).toBe('::0')
|
||||
expect(ipv6ToString({ part4: 1 } as any)).toBe('::1')
|
||||
})
|
||||
|
||||
it('skips missing latency and loss values', () => {
|
||||
expect(latencyMs(peerRoutePair([
|
||||
{ conn_id: 'missing', stats: {} },
|
||||
{ conn_id: 'valid', stats: { latency_us: '2500' } },
|
||||
{ conn_id: 'invalid', stats: { latency_us: 'unknown' } },
|
||||
]))).toBe('3ms')
|
||||
expect(latencyMs(peerRoutePair([
|
||||
{ conn_id: 'missing', stats: {} },
|
||||
{ conn_id: 'invalid', stats: { latency_us: 'unknown' } },
|
||||
]))).toBe('')
|
||||
|
||||
expect(lossRate(peerRoutePair([
|
||||
{ conn_id: 'missing' },
|
||||
{ conn_id: 'valid', loss_rate: '0.25' },
|
||||
{ conn_id: 'invalid', loss_rate: 'unknown' },
|
||||
]))).toBe('25%')
|
||||
expect(lossRate(peerRoutePair([
|
||||
{ conn_id: 'missing' },
|
||||
{ conn_id: 'invalid', loss_rate: 'unknown' },
|
||||
]))).toBe('')
|
||||
})
|
||||
|
||||
it('prefers the default connection when its metric is valid', () => {
|
||||
const defaultConnId = '00000001-0002-0003-0004-000000000005'
|
||||
const conns = [
|
||||
{ conn_id: 'fallback', stats: { latency_us: '1000' }, loss_rate: '0.01' },
|
||||
{ conn_id: defaultConnId, stats: { latency_us: '9000' }, loss_rate: '0.5' },
|
||||
]
|
||||
|
||||
expect(latencyMs(peerRoutePairWithDefaultConn(conns, defaultConnId))).toBe('9ms')
|
||||
expect(lossRate(peerRoutePairWithDefaultConn(conns, defaultConnId))).toBe('50%')
|
||||
})
|
||||
})
|
||||
@@ -218,7 +218,7 @@ class WebRemoteClient implements Api.RemoteClient {
|
||||
}
|
||||
async get_network_info(inst_id: string): Promise<NetworkTypes.NetworkInstanceRunningInfo | undefined> {
|
||||
const response = await this.client.get<any, Api.CollectNetworkInfoResponse>('/machines/' + this.machine_id + '/networks/info/' + inst_id);
|
||||
return response.info?.map?.[inst_id];
|
||||
return response.info.map[inst_id];
|
||||
}
|
||||
async list_network_instance_ids(): Promise<Api.ListNetworkInstanceIdResponse> {
|
||||
const response = await this.client.get<any, ListNetworkInstanceIdResponse>('/machines/' + this.machine_id + '/networks');
|
||||
|
||||
@@ -40,9 +40,6 @@ cli:
|
||||
geoip_db:
|
||||
en: "The path to the GeoIP2 database file, used to lookup the location of the client, default is the embedded file (only country information) , recommend https://github.com/P3TERX/GeoLite.mmdb"
|
||||
zh-CN: "GeoIP2 数据库文件路径,用于查找客户端的位置,默认为嵌入文件(仅国家信息),推荐 https://github.com/P3TERX/GeoLite.mmdb"
|
||||
heartbeat_min_response_ms:
|
||||
en: "Minimum response time for config-server heartbeat RPCs in milliseconds, default is 0"
|
||||
zh-CN: "配置服务心跳 RPC 的最短响应时间,单位毫秒,默认为 0"
|
||||
disable_registration:
|
||||
en: "Disable user registration"
|
||||
zh-CN: "禁用用户注册"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,3 @@
|
||||
mod managed_config;
|
||||
mod runtime_reconcile;
|
||||
pub mod session;
|
||||
pub mod storage;
|
||||
|
||||
@@ -7,7 +5,6 @@ use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU32, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
use dashmap::DashMap;
|
||||
use easytier::{
|
||||
@@ -33,10 +30,6 @@ use crate::db::{Db, UserIdInDb, entity::user_running_network_configs};
|
||||
#[include = "geoip2-cn.mmdb"]
|
||||
struct GeoipDb;
|
||||
|
||||
pub fn is_managed_config_revision_conflict(error: &anyhow::Error) -> bool {
|
||||
managed_config::is_revision_conflict(error)
|
||||
}
|
||||
|
||||
fn load_geoip_db(geoip_db: Option<String>) -> Option<maxminddb::Reader<Vec<u8>>> {
|
||||
if let Some(path) = geoip_db {
|
||||
match maxminddb::Reader::open_readfile(&path) {
|
||||
@@ -70,14 +63,12 @@ pub struct ClientManager {
|
||||
webhook_config: SharedWebhookConfig,
|
||||
|
||||
geoip_db: Arc<Option<maxminddb::Reader<Vec<u8>>>>,
|
||||
heartbeat_min_response_delay: Duration,
|
||||
}
|
||||
|
||||
impl ClientManager {
|
||||
pub fn new(
|
||||
db: Db,
|
||||
geoip_db: Option<String>,
|
||||
heartbeat_min_response_delay: Duration,
|
||||
feature_flags: Arc<FeatureFlags>,
|
||||
webhook_config: SharedWebhookConfig,
|
||||
) -> Self {
|
||||
@@ -101,7 +92,6 @@ impl ClientManager {
|
||||
webhook_config,
|
||||
|
||||
geoip_db: Arc::new(load_geoip_db(geoip_db)),
|
||||
heartbeat_min_response_delay,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +105,6 @@ impl ClientManager {
|
||||
let storage = self.storage.weak_ref();
|
||||
let listeners_cnt = self.listeners_cnt.clone();
|
||||
let geoip_db = self.geoip_db.clone();
|
||||
let heartbeat_min_response_delay = self.heartbeat_min_response_delay;
|
||||
let feature_flags = self.feature_flags.clone();
|
||||
let webhook_config = self.webhook_config.clone();
|
||||
self.tasks.spawn(async move {
|
||||
@@ -140,7 +129,6 @@ impl ClientManager {
|
||||
storage.clone(),
|
||||
client_url.clone(),
|
||||
location,
|
||||
heartbeat_min_response_delay,
|
||||
feature_flags.clone(),
|
||||
webhook_config.clone(),
|
||||
);
|
||||
@@ -161,10 +149,6 @@ impl ClientManager {
|
||||
self.storage.list_clients()
|
||||
}
|
||||
|
||||
pub async fn list_all_sessions(&self) -> Vec<StorageToken> {
|
||||
self.storage.list_all_clients()
|
||||
}
|
||||
|
||||
pub fn get_session_by_machine_id(
|
||||
&self,
|
||||
user_id: UserIdInDb,
|
||||
@@ -185,7 +169,7 @@ impl ClientManager {
|
||||
) -> bool {
|
||||
let Some(client_url) = self
|
||||
.storage
|
||||
.get_client_url_by_machine_id_with_auth(user_id, machine_id, false)
|
||||
.get_client_url_by_machine_id(user_id, machine_id)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
@@ -205,30 +189,14 @@ impl ClientManager {
|
||||
user_id: UserIdInDb,
|
||||
machine_id: uuid::Uuid,
|
||||
desired_configs: Vec<ManagedNetworkConfig>,
|
||||
config_revision: Option<String>,
|
||||
expected_config_revision: Option<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
let expected_config_revision = match expected_config_revision.as_deref().map(str::trim) {
|
||||
None => managed_config::ExpectedConfigRevision::Any,
|
||||
Some("") => managed_config::ExpectedConfigRevision::Exact(None),
|
||||
Some(revision) => managed_config::ExpectedConfigRevision::Exact(Some(revision)),
|
||||
};
|
||||
managed_config::reconcile_web_source_configs(
|
||||
session::SessionRpcService::reconcile_web_source_configs(
|
||||
&self.storage,
|
||||
user_id,
|
||||
machine_id,
|
||||
desired_configs,
|
||||
config_revision.as_deref(),
|
||||
expected_config_revision,
|
||||
)
|
||||
.await?;
|
||||
if let Some(config_revision) = config_revision
|
||||
&& let Some(session) = self.get_session_by_machine_id(user_id, &machine_id)
|
||||
{
|
||||
session
|
||||
.notify_config_revision_changed(user_id, machine_id, config_revision)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -363,449 +331,19 @@ impl
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use axum::{Json, Router, extract::State, routing::post};
|
||||
use easytier::{
|
||||
common::MachineIdOptions,
|
||||
instance_manager::NetworkInstanceManager,
|
||||
proto::{
|
||||
api::manage::{NetworkConfig, NetworkingMethod, PortForwardConfig},
|
||||
common::CompressionAlgoPb,
|
||||
},
|
||||
rpc_service::remote_client::Storage as RemoteStorage,
|
||||
tunnel::{
|
||||
common::tests::wait_for_condition,
|
||||
udp::{UdpTunnelConnector, UdpTunnelListener},
|
||||
},
|
||||
web_client::{WebClient, run_web_client},
|
||||
web_client::WebClient,
|
||||
};
|
||||
use serde_json::json;
|
||||
use sqlx::Executor;
|
||||
use tokio::net::UdpSocket;
|
||||
|
||||
use crate::{
|
||||
FeatureFlags, client_manager::ClientManager, db::Db, webhook::ManagedNetworkConfig,
|
||||
};
|
||||
|
||||
const MANAGED_CONFIG_TOKEN: &str = "managed-config-token";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TestWebhookState {
|
||||
validate_responses: Arc<tokio::sync::Mutex<VecDeque<bool>>>,
|
||||
validate_count: Arc<AtomicUsize>,
|
||||
block_second_validate: Arc<AtomicBool>,
|
||||
allow_second_validate: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl TestWebhookState {
|
||||
fn new(validate_responses: impl IntoIterator<Item = bool>) -> Self {
|
||||
Self {
|
||||
validate_responses: Arc::new(tokio::sync::Mutex::new(
|
||||
validate_responses.into_iter().collect(),
|
||||
)),
|
||||
validate_count: Arc::new(AtomicUsize::new(0)),
|
||||
block_second_validate: Arc::new(AtomicBool::new(false)),
|
||||
allow_second_validate: Arc::new(AtomicBool::new(true)),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_blocked_second_validate(
|
||||
validate_responses: impl IntoIterator<Item = bool>,
|
||||
) -> Self {
|
||||
let state = Self::new(validate_responses);
|
||||
state.block_second_validate.store(true, Ordering::Release);
|
||||
state.allow_second_validate.store(false, Ordering::Release);
|
||||
state
|
||||
}
|
||||
|
||||
fn allow_second_validate(&self) {
|
||||
self.allow_second_validate.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn validate_count(&self) -> usize {
|
||||
self.validate_count.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
async fn validate_token_handler(
|
||||
State(state): State<TestWebhookState>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let count = state.validate_count.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
if count == 2 && state.block_second_validate.load(Ordering::Acquire) {
|
||||
while !state.allow_second_validate.load(Ordering::Acquire) {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
let valid = state
|
||||
.validate_responses
|
||||
.lock()
|
||||
.await
|
||||
.pop_front()
|
||||
.unwrap_or(true);
|
||||
if !valid {
|
||||
return Json(json!({ "valid": false }));
|
||||
}
|
||||
|
||||
Json(json!({
|
||||
"valid": true,
|
||||
"binding_version": count,
|
||||
"config_revision": format!("validated-rev-{count}")
|
||||
}))
|
||||
}
|
||||
|
||||
async fn webhook_ack_handler() -> Json<serde_json::Value> {
|
||||
Json(json!({}))
|
||||
}
|
||||
|
||||
async fn test_webhook_config() -> (
|
||||
crate::webhook::SharedWebhookConfig,
|
||||
tokio::task::JoinHandle<()>,
|
||||
TestWebhookState,
|
||||
) {
|
||||
let state = TestWebhookState::new([true]);
|
||||
test_webhook_config_with_state(state).await
|
||||
}
|
||||
|
||||
async fn test_webhook_config_with_state(
|
||||
state: TestWebhookState,
|
||||
) -> (
|
||||
crate::webhook::SharedWebhookConfig,
|
||||
tokio::task::JoinHandle<()>,
|
||||
TestWebhookState,
|
||||
) {
|
||||
let app = Router::new()
|
||||
.route("/validate-token", post(validate_token_handler))
|
||||
.route("/webhook/node-connected", post(webhook_ack_handler))
|
||||
.route("/webhook/node-disconnected", post(webhook_ack_handler))
|
||||
.with_state(state.clone());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(
|
||||
Arc::new(crate::webhook::WebhookConfig::new(
|
||||
Some(format!("http://{addr}")),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)),
|
||||
server,
|
||||
state,
|
||||
)
|
||||
}
|
||||
|
||||
async fn add_random_udp_listener(mgr: &mut ClientManager) -> std::net::SocketAddr {
|
||||
let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap());
|
||||
let addr = socket.local_addr().unwrap();
|
||||
let listener =
|
||||
UdpTunnelListener::new_with_socket(format!("udp://{addr}").parse().unwrap(), socket);
|
||||
mgr.add_listener(listener).await.unwrap();
|
||||
addr
|
||||
}
|
||||
|
||||
async fn wait_for_validated_user(mgr: &ClientManager, machine_id: uuid::Uuid) -> i32 {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
if let Some(token) = mgr.list_sessions().await.into_iter().find(|token| {
|
||||
token.token == MANAGED_CONFIG_TOKEN && token.machine_id == machine_id
|
||||
}) {
|
||||
break token.user_id;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn wait_for_validate_count(state: &TestWebhookState, target: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
if state.validate_count() >= target {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn wait_for_session_urls(mgr: &ClientManager) -> Vec<url::Url> {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
let urls = mgr
|
||||
.client_sessions
|
||||
.iter()
|
||||
.map(|entry| entry.key().clone())
|
||||
.collect::<Vec<_>>();
|
||||
if !urls.is_empty() {
|
||||
break urls;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn managed_config(
|
||||
instance_id: uuid::Uuid,
|
||||
network_config: serde_json::Value,
|
||||
) -> ManagedNetworkConfig {
|
||||
ManagedNetworkConfig {
|
||||
instance_id: instance_id.to_string(),
|
||||
network_config,
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_runtime_config(
|
||||
manager: &NetworkInstanceManager,
|
||||
inst_id: uuid::Uuid,
|
||||
predicate: impl Fn(&NetworkConfig) -> bool,
|
||||
) -> NetworkConfig {
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
if let Some(config) = manager
|
||||
.get_instance_config(&inst_id)
|
||||
.and_then(|config| NetworkConfig::new_from_config(&config).ok())
|
||||
.filter(|config| predicate(config))
|
||||
{
|
||||
break config;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn start_web_client_for_test(
|
||||
config_server_addr: std::net::SocketAddr,
|
||||
machine_id: uuid::Uuid,
|
||||
manager: Arc<NetworkInstanceManager>,
|
||||
) -> WebClient {
|
||||
run_web_client(
|
||||
&format!("udp://{config_server_addr}/{MANAGED_CONFIG_TOKEN}"),
|
||||
MachineIdOptions {
|
||||
explicit_machine_id: Some(machine_id.to_string()),
|
||||
state_dir: None,
|
||||
},
|
||||
Some("managed-config-core".to_string()),
|
||||
false,
|
||||
manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn clear_managed_config_db(
|
||||
mgr: &ClientManager,
|
||||
user_id: i32,
|
||||
machine_id: uuid::Uuid,
|
||||
instance_id: uuid::Uuid,
|
||||
) {
|
||||
mgr.db()
|
||||
.delete_web_network_configs((user_id, machine_id), &[instance_id])
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("DELETE FROM managed_config_revisions WHERE user_id = ? AND device_id = ?")
|
||||
.bind(user_id)
|
||||
.bind(machine_id.to_string())
|
||||
.execute(&mgr.db().inner())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn assert_updated_runtime_config(updated: &NetworkConfig, instance_id: uuid::Uuid) {
|
||||
assert_eq!(
|
||||
updated.instance_id.as_deref(),
|
||||
Some(instance_id.to_string().as_str())
|
||||
);
|
||||
assert_eq!(updated.dhcp, Some(false));
|
||||
assert_eq!(updated.virtual_ipv4.as_deref(), Some("10.88.0.7"));
|
||||
assert_eq!(updated.network_length, Some(24));
|
||||
assert_eq!(updated.hostname.as_deref(), Some("managed-updated-host"));
|
||||
assert_eq!(updated.network_name.as_deref(), Some("managed-updated"));
|
||||
assert_eq!(updated.network_secret.as_deref(), Some("secret-updated"));
|
||||
assert_eq!(
|
||||
updated.networking_method,
|
||||
Some(NetworkingMethod::Manual as i32)
|
||||
);
|
||||
assert_eq!(updated.peer_urls, vec!["tcp://127.0.0.1:11010".to_string()]);
|
||||
assert_eq!(
|
||||
updated.proxy_cidrs,
|
||||
vec![
|
||||
"10.44.0.0/24".to_string(),
|
||||
"10.45.0.0/24->10.46.0.0/24".to_string()
|
||||
]
|
||||
);
|
||||
assert_eq!(updated.no_tun, Some(true));
|
||||
assert_eq!(updated.disable_ipv6, Some(true));
|
||||
assert_eq!(updated.enable_kcp_proxy, Some(true));
|
||||
assert_eq!(updated.disable_kcp_input, Some(true));
|
||||
assert_eq!(updated.enable_quic_proxy, Some(true));
|
||||
assert_eq!(updated.disable_quic_input, Some(true));
|
||||
assert_eq!(updated.disable_p2p, Some(true));
|
||||
assert_eq!(updated.p2p_only, Some(true));
|
||||
assert_eq!(updated.lazy_p2p, Some(true));
|
||||
assert_eq!(updated.relay_all_peer_rpc, Some(true));
|
||||
assert_eq!(updated.need_p2p, Some(true));
|
||||
assert_eq!(updated.multi_thread, Some(false));
|
||||
assert_eq!(updated.proxy_forward_by_system, Some(true));
|
||||
assert_eq!(updated.disable_encryption, Some(true));
|
||||
assert_eq!(updated.enable_relay_network_whitelist, Some(true));
|
||||
assert_eq!(
|
||||
updated.relay_network_whitelist,
|
||||
vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()]
|
||||
);
|
||||
assert_eq!(updated.enable_manual_routes, Some(true));
|
||||
assert_eq!(
|
||||
updated.routes,
|
||||
vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()]
|
||||
);
|
||||
assert_eq!(updated.port_forwards[0].bind_ip, "127.0.0.1");
|
||||
assert_eq!(updated.port_forwards[0].bind_port, 0);
|
||||
assert_eq!(updated.port_forwards[0].dst_ip, "10.88.0.8");
|
||||
assert_eq!(updated.port_forwards[0].dst_port, 80);
|
||||
assert_eq!(updated.port_forwards[0].proto, "tcp");
|
||||
assert_eq!(updated.disable_udp_hole_punching, Some(true));
|
||||
assert_eq!(updated.disable_tcp_hole_punching, Some(true));
|
||||
assert_eq!(updated.disable_sym_hole_punching, Some(true));
|
||||
assert_eq!(updated.disable_upnp, Some(true));
|
||||
assert_eq!(updated.disable_relay_data, Some(true));
|
||||
assert_eq!(updated.enable_magic_dns, Some(true));
|
||||
assert_eq!(updated.enable_private_mode, Some(true));
|
||||
assert_eq!(updated.mtu, Some(1360));
|
||||
assert_eq!(
|
||||
updated.data_compress_algo,
|
||||
Some(CompressionAlgoPb::Zstd as i32)
|
||||
);
|
||||
assert_eq!(updated.encryption_algorithm.as_deref(), Some("xor"));
|
||||
assert_eq!(updated.instance_recv_bps_limit, Some(123456));
|
||||
assert_eq!(updated.enable_udp_broadcast_relay, Some(true));
|
||||
assert_eq!(updated.socket_mark, Some(0));
|
||||
}
|
||||
|
||||
fn initial_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||
json!({
|
||||
"instance_id": inst_id.to_string(),
|
||||
"dhcp": true,
|
||||
"network_name": "managed-initial",
|
||||
"network_secret": "secret-initial",
|
||||
"networking_method": "Standalone",
|
||||
"no_tun": true,
|
||||
"disable_ipv6": true,
|
||||
"enable_kcp_proxy": false,
|
||||
"disable_kcp_input": false,
|
||||
"relay_all_peer_rpc": false,
|
||||
"multi_thread": false,
|
||||
"disable_relay_data": false,
|
||||
"mtu": 1380
|
||||
})
|
||||
}
|
||||
|
||||
fn updated_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||
serde_json::to_value(NetworkConfig {
|
||||
instance_id: Some(inst_id.to_string()),
|
||||
dhcp: Some(false),
|
||||
virtual_ipv4: Some("10.88.0.7".to_string()),
|
||||
network_length: Some(24),
|
||||
hostname: Some("managed-updated-host".to_string()),
|
||||
network_name: Some("managed-updated".to_string()),
|
||||
network_secret: Some("secret-updated".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
|
||||
proxy_cidrs: vec![
|
||||
"10.44.0.0/24".to_string(),
|
||||
"10.45.0.0/24->10.46.0.0/24".to_string(),
|
||||
],
|
||||
no_tun: Some(true),
|
||||
disable_ipv6: Some(true),
|
||||
enable_kcp_proxy: Some(true),
|
||||
disable_kcp_input: Some(true),
|
||||
enable_quic_proxy: Some(true),
|
||||
disable_quic_input: Some(true),
|
||||
disable_p2p: Some(true),
|
||||
p2p_only: Some(true),
|
||||
lazy_p2p: Some(true),
|
||||
relay_all_peer_rpc: Some(true),
|
||||
need_p2p: Some(true),
|
||||
multi_thread: Some(false),
|
||||
proxy_forward_by_system: Some(true),
|
||||
disable_encryption: Some(true),
|
||||
enable_relay_network_whitelist: Some(true),
|
||||
relay_network_whitelist: vec!["10.44.0.0/24".to_string(), "10.45.0.0/24".to_string()],
|
||||
enable_manual_routes: Some(true),
|
||||
routes: vec!["10.60.0.0/16".to_string(), "10.61.0.0/16".to_string()],
|
||||
port_forwards: vec![PortForwardConfig {
|
||||
bind_ip: "127.0.0.1".to_string(),
|
||||
bind_port: 0,
|
||||
dst_ip: "10.88.0.8".to_string(),
|
||||
dst_port: 80,
|
||||
proto: "tcp".to_string(),
|
||||
}],
|
||||
disable_udp_hole_punching: Some(true),
|
||||
disable_tcp_hole_punching: Some(true),
|
||||
disable_sym_hole_punching: Some(true),
|
||||
disable_upnp: Some(true),
|
||||
disable_relay_data: Some(true),
|
||||
enable_magic_dns: Some(true),
|
||||
enable_private_mode: Some(true),
|
||||
mtu: Some(1360),
|
||||
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
|
||||
encryption_algorithm: Some("xor".to_string()),
|
||||
instance_recv_bps_limit: Some(123456),
|
||||
enable_udp_broadcast_relay: Some(true),
|
||||
socket_mark: Some(0),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn redelivered_managed_network_config(inst_id: uuid::Uuid) -> serde_json::Value {
|
||||
serde_json::to_value(NetworkConfig {
|
||||
instance_id: Some(inst_id.to_string()),
|
||||
dhcp: Some(false),
|
||||
virtual_ipv4: Some("10.88.0.7".to_string()),
|
||||
network_length: Some(24),
|
||||
hostname: Some("managed-redelivered-host".to_string()),
|
||||
network_name: Some("managed-redelivered".to_string()),
|
||||
network_secret: Some("secret-updated".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
peer_urls: vec!["tcp://127.0.0.1:11010".to_string()],
|
||||
proxy_cidrs: vec![
|
||||
"10.44.0.0/24".to_string(),
|
||||
"10.45.0.0/24->10.46.0.0/24".to_string(),
|
||||
],
|
||||
no_tun: Some(true),
|
||||
disable_ipv6: Some(true),
|
||||
enable_kcp_proxy: Some(true),
|
||||
disable_kcp_input: Some(true),
|
||||
relay_all_peer_rpc: Some(true),
|
||||
need_p2p: Some(true),
|
||||
multi_thread: Some(false),
|
||||
enable_private_mode: Some(true),
|
||||
mtu: Some(1360),
|
||||
data_compress_algo: Some(CompressionAlgoPb::Zstd as i32),
|
||||
encryption_algorithm: Some("xor".to_string()),
|
||||
instance_recv_bps_limit: Some(654321),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
use crate::{FeatureFlags, client_manager::ClientManager, db::Db};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_client() {
|
||||
@@ -813,7 +351,6 @@ mod tests {
|
||||
let mut mgr = ClientManager::new(
|
||||
Db::memory_db().await,
|
||||
None,
|
||||
Duration::ZERO,
|
||||
Arc::new(FeatureFlags::default()),
|
||||
Arc::new(crate::webhook::WebhookConfig::new(
|
||||
None, None, None, None, None,
|
||||
@@ -873,223 +410,4 @@ mod tests {
|
||||
println!("{:?}", req);
|
||||
println!("{:?}", mgr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn managed_web_config_revision_updates_running_core_config() {
|
||||
let (webhook_config, webhook_server, _) = test_webhook_config().await;
|
||||
let mut mgr = ClientManager::new(
|
||||
Db::memory_db().await,
|
||||
None,
|
||||
Duration::ZERO,
|
||||
Arc::new(FeatureFlags::default()),
|
||||
webhook_config,
|
||||
);
|
||||
let config_server_addr = add_random_udp_listener(&mut mgr).await;
|
||||
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let instance_id = uuid::Uuid::new_v4();
|
||||
let core_manager = Arc::new(NetworkInstanceManager::new());
|
||||
let client =
|
||||
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
|
||||
|
||||
let user_id = wait_for_validated_user(&mgr, machine_id).await;
|
||||
mgr.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
vec![managed_config(
|
||||
instance_id,
|
||||
initial_managed_network_config(instance_id),
|
||||
)],
|
||||
Some("rev-initial".to_string()),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-initial")
|
||||
})
|
||||
.await;
|
||||
|
||||
// Online revision update: web-owned running config is fully overwritten
|
||||
// when non-hot-patch flags such as enable_kcp_proxy change.
|
||||
mgr.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
vec![managed_config(
|
||||
instance_id,
|
||||
updated_managed_network_config(instance_id),
|
||||
)],
|
||||
Some("rev-updated".to_string()),
|
||||
Some("rev-initial".to_string()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let updated = wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-updated")
|
||||
&& config.enable_kcp_proxy == Some(true)
|
||||
&& config.port_forwards.len() == 1
|
||||
})
|
||||
.await;
|
||||
assert_updated_runtime_config(&updated, instance_id);
|
||||
|
||||
assert_eq!(
|
||||
core_manager.get_instance_network_config_source(&instance_id),
|
||||
Some(easytier::common::config::ConfigSource::Web)
|
||||
);
|
||||
assert_eq!(
|
||||
mgr.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("rev-updated")
|
||||
);
|
||||
|
||||
// Web DB loss path: clear web-owned config and revision, then simulate
|
||||
// the webhook re-posting the authoritative desired config. The already
|
||||
// connected session should receive the distinguishable re-delivered
|
||||
// revision without restarting.
|
||||
clear_managed_config_db(&mgr, user_id, machine_id, instance_id).await;
|
||||
assert!(
|
||||
mgr.db()
|
||||
.get_network_config((user_id, machine_id), &instance_id.to_string())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
mgr.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
mgr.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
vec![managed_config(
|
||||
instance_id,
|
||||
redelivered_managed_network_config(instance_id),
|
||||
)],
|
||||
Some("rev-webhook-redelivery".to_string()),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let redelivered = wait_for_runtime_config(&core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-redelivered")
|
||||
&& config.instance_recv_bps_limit == Some(654321)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
redelivered.instance_id.as_deref(),
|
||||
Some(instance_id.to_string().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
redelivered.hostname.as_deref(),
|
||||
Some("managed-redelivered-host")
|
||||
);
|
||||
assert_eq!(
|
||||
redelivered.network_name.as_deref(),
|
||||
Some("managed-redelivered")
|
||||
);
|
||||
assert_eq!(redelivered.enable_kcp_proxy, Some(true));
|
||||
assert_eq!(redelivered.instance_recv_bps_limit, Some(654321));
|
||||
assert_eq!(
|
||||
core_manager.get_instance_network_config_source(&instance_id),
|
||||
Some(easytier::common::config::ConfigSource::Web)
|
||||
);
|
||||
assert_eq!(
|
||||
mgr.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("rev-webhook-redelivery")
|
||||
);
|
||||
|
||||
// Reconnect path: a fresh core manager has no local runtime state, so
|
||||
// the new session must replay the managed config persisted in web DB.
|
||||
drop(client);
|
||||
let reconnected_core_manager = Arc::new(NetworkInstanceManager::new());
|
||||
let _reconnected_client = start_web_client_for_test(
|
||||
config_server_addr,
|
||||
machine_id,
|
||||
reconnected_core_manager.clone(),
|
||||
)
|
||||
.await;
|
||||
wait_for_validated_user(&mgr, machine_id).await;
|
||||
let replayed = wait_for_runtime_config(&reconnected_core_manager, instance_id, |config| {
|
||||
config.network_name.as_deref() == Some("managed-redelivered")
|
||||
&& config.instance_recv_bps_limit == Some(654321)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
replayed.network_name.as_deref(),
|
||||
Some("managed-redelivered")
|
||||
);
|
||||
assert_eq!(replayed.enable_kcp_proxy, Some(true));
|
||||
assert_eq!(replayed.instance_recv_bps_limit, Some(654321));
|
||||
|
||||
webhook_server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webhook_reject_disconnects_and_revalidates_after_reconnect() {
|
||||
let webhook_state = TestWebhookState::with_blocked_second_validate([false, true]);
|
||||
let (webhook_config, webhook_server, webhook_state) =
|
||||
test_webhook_config_with_state(webhook_state).await;
|
||||
let mut mgr = ClientManager::new(
|
||||
Db::memory_db().await,
|
||||
None,
|
||||
Duration::ZERO,
|
||||
Arc::new(FeatureFlags::default()),
|
||||
webhook_config,
|
||||
);
|
||||
let config_server_addr = add_random_udp_listener(&mut mgr).await;
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let core_manager = Arc::new(NetworkInstanceManager::new());
|
||||
let client =
|
||||
start_web_client_for_test(config_server_addr, machine_id, core_manager.clone()).await;
|
||||
|
||||
let first_session_urls = wait_for_session_urls(&mgr).await;
|
||||
wait_for_validate_count(&webhook_state, 1).await;
|
||||
wait_for_validate_count(&webhook_state, 2).await;
|
||||
assert!(
|
||||
mgr.list_sessions().await.is_empty(),
|
||||
"invalid validate-token response must not authorize the session"
|
||||
);
|
||||
|
||||
webhook_state.allow_second_validate();
|
||||
let user_id = wait_for_validated_user(&mgr, machine_id).await;
|
||||
tokio::time::timeout(Duration::from_secs(12), async {
|
||||
loop {
|
||||
let reconnected = mgr
|
||||
.client_sessions
|
||||
.iter()
|
||||
.any(|entry| !first_session_urls.iter().any(|url| url == entry.key()));
|
||||
if reconnected {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
client.is_connected(),
|
||||
"web client should reconnect after invalid session heartbeat failure"
|
||||
);
|
||||
assert!(webhook_state.validate_count() >= 2);
|
||||
assert!(
|
||||
mgr.get_session_by_machine_id(user_id, &machine_id)
|
||||
.is_some()
|
||||
);
|
||||
|
||||
webhook_server.abort();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,812 +0,0 @@
|
||||
use anyhow::Context as _;
|
||||
use easytier::{
|
||||
common::config::{
|
||||
ConfigLoader, EncryptionAlgorithm, PortForwardConfig as RuntimePortForwardConfig,
|
||||
},
|
||||
proto::{
|
||||
acl::Acl,
|
||||
api::{
|
||||
config::{
|
||||
AclPatch, ConfigPatchAction, InstanceConfigPatch, PatchConfigRequest,
|
||||
PortForwardPatch, ProxyNetworkPatch,
|
||||
},
|
||||
instance::{InstanceIdentifier, instance_identifier},
|
||||
manage::{
|
||||
ConfigSource as RpcConfigSource, GetNetworkInstanceConfigRequest, NetworkConfig,
|
||||
RunNetworkInstanceRequest,
|
||||
},
|
||||
},
|
||||
common::{CompressionAlgoPb, Ipv4Inet as RpcIpv4Inet},
|
||||
rpc_types::controller::BaseController,
|
||||
},
|
||||
};
|
||||
|
||||
use super::session::{SessionConfigClient, SessionRpcClient};
|
||||
|
||||
pub(super) enum RuntimeReconcileAction {
|
||||
None,
|
||||
Run {
|
||||
config: Box<NetworkConfig>,
|
||||
overwrite: bool,
|
||||
},
|
||||
Patch(Box<InstanceConfigPatch>),
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq)]
|
||||
struct RuntimeProxyNetwork {
|
||||
cidr: String,
|
||||
mapped_cidr: Option<String>,
|
||||
}
|
||||
|
||||
fn instance_identifier(inst_id: &str) -> anyhow::Result<InstanceIdentifier> {
|
||||
let inst_id = uuid::Uuid::parse_str(inst_id)
|
||||
.with_context(|| format!("invalid runtime instance id: {inst_id}"))?;
|
||||
Ok(InstanceIdentifier {
|
||||
selector: Some(instance_identifier::Selector::Id(inst_id.into())),
|
||||
})
|
||||
}
|
||||
|
||||
fn hot_patch_base(config: &NetworkConfig) -> anyhow::Result<NetworkConfig> {
|
||||
let data_compress_algo = normalized_data_compress_algo(config.data_compress_algo);
|
||||
let encryption_algorithm = normalized_encryption_algorithm(config.encryption_algorithm.clone());
|
||||
let mut config = NetworkConfig::new_from_config(config.gen_config()?)?;
|
||||
let is_credential_mode = config.network_secret.is_none()
|
||||
&& config
|
||||
.secure_mode
|
||||
.as_ref()
|
||||
.and_then(|mode| mode.local_private_key.as_deref())
|
||||
.is_some_and(|key| !key.is_empty());
|
||||
config.acl = None;
|
||||
config.port_forwards.clear();
|
||||
config.proxy_cidrs.clear();
|
||||
config.disable_relay_data = None;
|
||||
if config.dhcp.unwrap_or_default() {
|
||||
config.virtual_ipv4 = None;
|
||||
config.network_length = None;
|
||||
}
|
||||
if let Some(secure_mode) = config.secure_mode.as_mut() {
|
||||
if !is_credential_mode {
|
||||
secure_mode.local_private_key = None;
|
||||
}
|
||||
secure_mode.local_public_key = None;
|
||||
}
|
||||
config.data_compress_algo = data_compress_algo;
|
||||
config.encryption_algorithm = encryption_algorithm;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn normalized_data_compress_algo(algo: Option<i32>) -> Option<i32> {
|
||||
let default = CompressionAlgoPb::None as i32;
|
||||
let effective = algo.map(|algo| if algo < default { default } else { algo });
|
||||
effective.filter(|algo| *algo != default)
|
||||
}
|
||||
|
||||
fn normalized_encryption_algorithm(algo: Option<String>) -> Option<String> {
|
||||
let default = EncryptionAlgorithm::default().to_string();
|
||||
algo.filter(|algo| algo != &default)
|
||||
}
|
||||
|
||||
fn diff_port_forwards(
|
||||
current: &[RuntimePortForwardConfig],
|
||||
desired: &[RuntimePortForwardConfig],
|
||||
) -> Vec<PortForwardPatch> {
|
||||
let mut patches = Vec::new();
|
||||
for cfg in unique_port_forwards(current, desired) {
|
||||
let current_count = current.iter().filter(|item| *item == &cfg).count();
|
||||
let desired_count = desired.iter().filter(|item| *item == &cfg).count();
|
||||
if current_count == desired_count {
|
||||
continue;
|
||||
}
|
||||
if current_count > 0 {
|
||||
patches.push(PortForwardPatch {
|
||||
action: ConfigPatchAction::Remove as i32,
|
||||
cfg: Some(cfg.clone().into()),
|
||||
});
|
||||
}
|
||||
patches.extend((0..desired_count).map(|_| PortForwardPatch {
|
||||
action: ConfigPatchAction::Add as i32,
|
||||
cfg: Some(cfg.clone().into()),
|
||||
}));
|
||||
}
|
||||
patches
|
||||
}
|
||||
|
||||
fn unique_port_forwards(
|
||||
current: &[RuntimePortForwardConfig],
|
||||
desired: &[RuntimePortForwardConfig],
|
||||
) -> Vec<RuntimePortForwardConfig> {
|
||||
let mut unique = Vec::new();
|
||||
for cfg in current.iter().chain(desired.iter()) {
|
||||
if !unique.contains(cfg) {
|
||||
unique.push(cfg.clone());
|
||||
}
|
||||
}
|
||||
unique
|
||||
}
|
||||
|
||||
fn parse_rpc_ipv4_inet(value: &str) -> anyhow::Result<RpcIpv4Inet> {
|
||||
value
|
||||
.parse::<RpcIpv4Inet>()
|
||||
.with_context(|| format!("failed to parse runtime ipv4 cidr: {value}"))
|
||||
}
|
||||
|
||||
fn diff_proxy_networks(
|
||||
current: &[RuntimeProxyNetwork],
|
||||
desired: &[RuntimeProxyNetwork],
|
||||
) -> anyhow::Result<Vec<ProxyNetworkPatch>> {
|
||||
if current == desired {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut patches = vec![ProxyNetworkPatch {
|
||||
action: ConfigPatchAction::Clear as i32,
|
||||
cidr: Some(clear_proxy_network_cidr(current, desired)?),
|
||||
..Default::default()
|
||||
}];
|
||||
for proxy_network in desired {
|
||||
patches.push(ProxyNetworkPatch {
|
||||
action: ConfigPatchAction::Add as i32,
|
||||
cidr: Some(parse_rpc_ipv4_inet(&proxy_network.cidr)?),
|
||||
mapped_cidr: proxy_network
|
||||
.mapped_cidr
|
||||
.as_deref()
|
||||
.map(parse_rpc_ipv4_inet)
|
||||
.transpose()?,
|
||||
});
|
||||
}
|
||||
Ok(patches)
|
||||
}
|
||||
|
||||
fn clear_proxy_network_cidr(
|
||||
current: &[RuntimeProxyNetwork],
|
||||
desired: &[RuntimeProxyNetwork],
|
||||
) -> anyhow::Result<RpcIpv4Inet> {
|
||||
let cidr = desired
|
||||
.first()
|
||||
.or_else(|| current.first())
|
||||
.map(|proxy_network| proxy_network.cidr.as_str())
|
||||
.unwrap_or("0.0.0.0/0");
|
||||
parse_rpc_ipv4_inet(cidr)
|
||||
}
|
||||
|
||||
fn normalized_acl(acl: &Option<Acl>) -> Option<Acl> {
|
||||
let acl = acl.clone().unwrap_or_default();
|
||||
(acl != Acl::default()).then_some(acl)
|
||||
}
|
||||
|
||||
fn normalized_port_forwards(
|
||||
config: &NetworkConfig,
|
||||
) -> anyhow::Result<Vec<RuntimePortForwardConfig>> {
|
||||
Ok(config
|
||||
.gen_config()?
|
||||
.get_port_forwards()
|
||||
.into_iter()
|
||||
.map(|cfg| {
|
||||
RuntimePortForwardConfig::from(easytier::proto::common::PortForwardConfigPb::from(cfg))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn normalized_proxy_networks(config: &NetworkConfig) -> anyhow::Result<Vec<RuntimeProxyNetwork>> {
|
||||
Ok(config
|
||||
.gen_config()?
|
||||
.get_proxy_cidrs()
|
||||
.into_iter()
|
||||
.map(|proxy_network| RuntimeProxyNetwork {
|
||||
cidr: proxy_network.cidr.to_string(),
|
||||
mapped_cidr: proxy_network.mapped_cidr.map(|cidr| cidr.to_string()),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn normalized_disable_relay_data(config: &NetworkConfig) -> anyhow::Result<bool> {
|
||||
Ok(config.gen_config()?.get_flags().disable_relay_data)
|
||||
}
|
||||
|
||||
fn web_source_runtime_patch(
|
||||
current: &NetworkConfig,
|
||||
desired: &NetworkConfig,
|
||||
) -> anyhow::Result<Option<InstanceConfigPatch>> {
|
||||
if let Some(desired_hostname) = desired
|
||||
.hostname
|
||||
.as_deref()
|
||||
.filter(|hostname| !hostname.is_empty())
|
||||
&& current.hostname.as_deref() != Some(desired_hostname)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let mut current_base = hot_patch_base(current)?;
|
||||
let mut desired_base = hot_patch_base(desired)?;
|
||||
current_base.hostname = None;
|
||||
desired_base.hostname = None;
|
||||
if current_base != desired_base {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut patch = InstanceConfigPatch::default();
|
||||
let current_acl = normalized_acl(¤t.acl);
|
||||
let desired_acl = normalized_acl(&desired.acl);
|
||||
if current_acl != desired_acl {
|
||||
patch.acl = Some(AclPatch {
|
||||
acl: Some(desired_acl.unwrap_or_default()),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
let current_port_forwards = normalized_port_forwards(current)?;
|
||||
let desired_port_forwards = normalized_port_forwards(desired)?;
|
||||
if current_port_forwards != desired_port_forwards {
|
||||
patch.port_forwards = diff_port_forwards(¤t_port_forwards, &desired_port_forwards);
|
||||
}
|
||||
|
||||
let current_proxy_networks = normalized_proxy_networks(current)?;
|
||||
let desired_proxy_networks = normalized_proxy_networks(desired)?;
|
||||
if current_proxy_networks != desired_proxy_networks {
|
||||
if current_proxy_networks.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
patch.proxy_networks =
|
||||
diff_proxy_networks(¤t_proxy_networks, &desired_proxy_networks)?;
|
||||
}
|
||||
|
||||
let current_disable_relay_data = normalized_disable_relay_data(current)?;
|
||||
let desired_disable_relay_data = normalized_disable_relay_data(desired)?;
|
||||
if current_disable_relay_data != desired_disable_relay_data {
|
||||
patch.disable_relay_data = Some(desired_disable_relay_data);
|
||||
}
|
||||
|
||||
Ok(Some(patch))
|
||||
}
|
||||
|
||||
fn ensure_runtime_config_converged(
|
||||
current: &NetworkConfig,
|
||||
desired: &NetworkConfig,
|
||||
) -> anyhow::Result<()> {
|
||||
let patch = web_source_runtime_patch(current, desired)?;
|
||||
match patch {
|
||||
Some(patch) if patch == InstanceConfigPatch::default() => Ok(()),
|
||||
Some(patch) => anyhow::bail!("runtime config still needs patch after reconcile: {patch:?}"),
|
||||
None => anyhow::bail!("runtime config still needs full overwrite after reconcile"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_web_source_instance(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
config: NetworkConfig,
|
||||
overwrite: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
rpc_client
|
||||
.run_network_instance(
|
||||
BaseController::default(),
|
||||
RunNetworkInstanceRequest {
|
||||
inst_id: Some(inst_id.to_string().into()),
|
||||
config: Some(config),
|
||||
overwrite,
|
||||
source: RpcConfigSource::Web as i32,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn get_runtime_config(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
) -> anyhow::Result<NetworkConfig> {
|
||||
rpc_client
|
||||
.get_network_instance_config(
|
||||
BaseController::default(),
|
||||
GetNetworkInstanceConfigRequest {
|
||||
inst_id: Some(inst_id.to_string().into()),
|
||||
},
|
||||
)
|
||||
.await?
|
||||
.config
|
||||
.ok_or_else(|| anyhow::anyhow!("runtime returned empty config for {inst_id}"))
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_web_source_runtime_reconcile(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
desired_config: NetworkConfig,
|
||||
is_running: bool,
|
||||
) -> anyhow::Result<RuntimeReconcileAction> {
|
||||
if !is_running {
|
||||
return Ok(RuntimeReconcileAction::Run {
|
||||
config: Box::new(desired_config),
|
||||
overwrite: false,
|
||||
});
|
||||
}
|
||||
|
||||
let current_config = get_runtime_config(rpc_client, inst_id).await?;
|
||||
|
||||
prepare_web_source_runtime_reconcile_from_current(¤t_config, desired_config)
|
||||
}
|
||||
|
||||
pub(super) fn prepare_web_source_runtime_reconcile_from_current(
|
||||
current_config: &NetworkConfig,
|
||||
desired_config: NetworkConfig,
|
||||
) -> anyhow::Result<RuntimeReconcileAction> {
|
||||
let Some(patch) = web_source_runtime_patch(current_config, &desired_config)? else {
|
||||
return Ok(RuntimeReconcileAction::Run {
|
||||
config: Box::new(desired_config),
|
||||
overwrite: true,
|
||||
});
|
||||
};
|
||||
if patch == InstanceConfigPatch::default() {
|
||||
return Ok(RuntimeReconcileAction::None);
|
||||
}
|
||||
|
||||
Ok(RuntimeReconcileAction::Patch(Box::new(patch)))
|
||||
}
|
||||
|
||||
pub(super) async fn apply_web_source_runtime_reconcile(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
config_client: &mut SessionConfigClient,
|
||||
inst_id: &str,
|
||||
desired_config: NetworkConfig,
|
||||
action: RuntimeReconcileAction,
|
||||
) -> anyhow::Result<NetworkConfig> {
|
||||
match action {
|
||||
RuntimeReconcileAction::None => Ok(desired_config),
|
||||
RuntimeReconcileAction::Run { config, overwrite } => {
|
||||
run_web_source_instance(rpc_client, inst_id, *config, overwrite).await?;
|
||||
Ok(desired_config)
|
||||
}
|
||||
RuntimeReconcileAction::Patch(patch) => {
|
||||
config_client
|
||||
.patch_config(
|
||||
BaseController::default(),
|
||||
PatchConfigRequest {
|
||||
instance: Some(instance_identifier(inst_id)?),
|
||||
patch: Some(*patch),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let current_config = get_runtime_config(rpc_client, inst_id).await?;
|
||||
ensure_runtime_config_converged(¤t_config, &desired_config)?;
|
||||
Ok(current_config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use easytier::proto::{
|
||||
api::{
|
||||
config::ConfigPatchAction,
|
||||
manage::{NetworkingMethod, PortForwardConfig},
|
||||
},
|
||||
common::{CompressionAlgoPb, SocketType},
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
|
||||
NetworkConfig {
|
||||
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
|
||||
dhcp: Some(true),
|
||||
network_name: Some("managed".to_string()),
|
||||
network_secret: Some("secret".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
port_forwards,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
|
||||
PortForwardConfig {
|
||||
bind_ip: "127.0.0.1".to_string(),
|
||||
bind_port,
|
||||
dst_ip: "10.144.0.1".to_string(),
|
||||
dst_port,
|
||||
proto: "tcp".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn patch_port(patch: &PortForwardPatch) -> (i32, u32, u32, i32) {
|
||||
let cfg = patch.cfg.as_ref().expect("port forward patch cfg");
|
||||
(
|
||||
patch.action,
|
||||
cfg.bind_addr.as_ref().expect("bind addr").port,
|
||||
cfg.dst_addr.as_ref().expect("dst addr").port,
|
||||
cfg.socket_type,
|
||||
)
|
||||
}
|
||||
|
||||
fn patch_proxy_network(patch: &ProxyNetworkPatch) -> (i32, String, Option<String>) {
|
||||
(
|
||||
patch.action,
|
||||
patch.cidr.map(|cidr| cidr.to_string()).unwrap_or_default(),
|
||||
patch.mapped_cidr.map(|cidr| cidr.to_string()),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_ignores_runtime_defaults_and_adds_port_forward() {
|
||||
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
current.virtual_ipv4 = Some("10.144.0.2".to_string());
|
||||
current.network_length = Some(16);
|
||||
current.bind_device = Some(true);
|
||||
current.dev_name = Some(String::new());
|
||||
current.disable_ipv6 = Some(false);
|
||||
current.mtu = Some(1380);
|
||||
current.multi_thread = Some(true);
|
||||
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_removes_deleted_port_forward_without_clear() {
|
||||
let current =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Remove as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_reconciles_duplicate_port_forward_count() {
|
||||
let current =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23000, 5174)]);
|
||||
let desired = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 2);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Remove as i32,
|
||||
23000,
|
||||
5174,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[1]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23000,
|
||||
5174,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_convergence_rejects_stale_extra_port_forward() {
|
||||
let current = config_with_port_forwards(vec![
|
||||
port_forward(23000, 5174),
|
||||
port_forward(23007, 3389),
|
||||
port_forward(23100, 8080),
|
||||
]);
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let err = ensure_runtime_config_converged(¤t, &desired)
|
||||
.expect_err("extra runtime port forward should not converge");
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("runtime config still needs patch after reconcile"),
|
||||
"unexpected error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_canonicalizes_port_forward_protocol() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired_port_forward = port_forward(23000, 5174);
|
||||
desired_port_forward.proto = "TCP".to_string();
|
||||
let desired = config_with_port_forwards(vec![desired_port_forward]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch, InstanceConfigPatch::default());
|
||||
ensure_runtime_config_converged(¤t, &desired).expect("runtime converged");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_non_hot_config_change() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = current.clone();
|
||||
desired.network_secret = Some("new-secret".to_string());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_routes_change() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.enable_manual_routes = Some(true);
|
||||
current.routes = vec!["10.1.0.0/16".to_string(), "10.2.0.0/16".to_string()];
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.enable_manual_routes = Some(true);
|
||||
desired.routes = vec!["10.2.0.0/16".to_string(), "10.3.0.0/16".to_string()];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_replaces_proxy_networks() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.proxy_cidrs = vec![
|
||||
"10.1.0.0/16".to_string(),
|
||||
"10.2.0.0/16->10.20.0.0/16".to_string(),
|
||||
];
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.proxy_cidrs = vec![
|
||||
"10.2.0.0/16->10.21.0.0/16".to_string(),
|
||||
"10.3.0.0/16".to_string(),
|
||||
];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.proxy_networks.len(), 3);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[0]),
|
||||
(
|
||||
ConfigPatchAction::Clear as i32,
|
||||
"10.2.0.0/16".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[1]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
"10.2.0.0/16".to_string(),
|
||||
Some("10.21.0.0/16".to_string())
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[2]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
"10.3.0.0/16".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_replaces_proxy_networks_with_same_source_cidr() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.proxy_cidrs = vec![
|
||||
"10.1.2.0/24".to_string(),
|
||||
"10.1.2.0/24->10.1.3.0/24".to_string(),
|
||||
];
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.proxy_cidrs = vec!["10.1.2.0/24->10.1.3.0/24".to_string()];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.proxy_networks.len(), 2);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[0]),
|
||||
(
|
||||
ConfigPatchAction::Clear as i32,
|
||||
"10.1.2.0/24".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[1]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
"10.1.2.0/24".to_string(),
|
||||
Some("10.1.3.0/24".to_string())
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_proxy_network_empty_to_nonempty() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = config_with_port_forwards(Vec::new());
|
||||
desired.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_clears_proxy_networks_with_legacy_compatible_cidr() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.proxy_cidrs = vec!["10.1.2.0/24".to_string()];
|
||||
let desired = config_with_port_forwards(Vec::new());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.proxy_networks.len(), 1);
|
||||
assert_eq!(
|
||||
patch_proxy_network(&patch.proxy_networks[0]),
|
||||
(
|
||||
ConfigPatchAction::Clear as i32,
|
||||
"10.1.2.0/24".to_string(),
|
||||
None
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_updates_disable_relay_data() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = current.clone();
|
||||
desired.disable_relay_data = Some(true);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.disable_relay_data, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_still_rejects_unsupported_flag_change() {
|
||||
let current = config_with_port_forwards(Vec::new());
|
||||
let mut desired = current.clone();
|
||||
desired.no_tun = Some(true);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_encryption_algorithm_change() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired = current.clone();
|
||||
desired.encryption_algorithm = Some("managed-test-algo".to_string());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_data_compress_algo_change() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired = current.clone();
|
||||
desired.data_compress_algo = Some(CompressionAlgoPb::Zstd as i32);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_credential_private_key_change() {
|
||||
let mut current = config_with_port_forwards(Vec::new());
|
||||
current.network_secret = None;
|
||||
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
|
||||
local_public_key: None,
|
||||
});
|
||||
let mut desired = current.clone();
|
||||
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: Some("aEpz80FuYbaY4QLJizAIuIcK4TYsoSA9jHHCXCOQJoc=".to_string()),
|
||||
local_public_key: None,
|
||||
});
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_ignores_generated_secure_key_when_network_secret_exists() {
|
||||
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
current.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: Some("mUuD5fsIm/ftvgS4WBAYFMNLqWX3qT9rnm4PrnOqb9s=".to_string()),
|
||||
local_public_key: Some("4x6L5dZjB8hsPO4f96Hyhi4xFealBu6i3BxRVBYR1Fc=".to_string()),
|
||||
});
|
||||
let mut desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
desired.secure_mode = Some(easytier::proto::common::SecureModeConfig {
|
||||
enabled: true,
|
||||
local_private_key: None,
|
||||
local_public_key: None,
|
||||
});
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_ignores_runtime_hostname_when_desired_omits_hostname() {
|
||||
let mut current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
current.hostname = Some("runtime-host".to_string());
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired)
|
||||
.expect("build patch")
|
||||
.expect("hot patch");
|
||||
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
assert_eq!(
|
||||
patch_port(&patch.port_forwards[0]),
|
||||
(
|
||||
ConfigPatchAction::Add as i32,
|
||||
23007,
|
||||
3389,
|
||||
SocketType::Tcp as i32
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_patch_rejects_explicit_desired_hostname_change() {
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let mut desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
desired.hostname =
|
||||
Some(easytier::common::config::TomlConfigLoader::default().get_hostname());
|
||||
|
||||
let patch = web_source_runtime_patch(¤t, &desired).expect("build patch");
|
||||
|
||||
assert!(patch.is_none());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,913 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use easytier::{
|
||||
proto::{
|
||||
api::manage::{
|
||||
DeleteNetworkInstanceRequest, ListNetworkInstanceMetaRequest,
|
||||
ListNetworkInstanceRequest, NetworkConfig, NetworkMeta, RunNetworkInstanceRequest,
|
||||
},
|
||||
rpc_types::controller::BaseController,
|
||||
web::HeartbeatRequest,
|
||||
},
|
||||
rpc_service::remote_client::{ListNetworkProps, Storage as _},
|
||||
};
|
||||
use tokio::sync::{RwLock, broadcast};
|
||||
|
||||
use super::{SessionConfigClient, SessionData, SessionRpcClient, SessionRpcService};
|
||||
use crate::client_manager::{
|
||||
managed_config::{self, PersistedConfigSource},
|
||||
runtime_reconcile,
|
||||
storage::{StorageInner, WeakRefStorage},
|
||||
};
|
||||
|
||||
async fn recv_latest_heartbeat(
|
||||
heartbeat_waiter: &mut broadcast::Receiver<HeartbeatRequest>,
|
||||
) -> Option<HeartbeatRequest> {
|
||||
let mut req = loop {
|
||||
match heartbeat_waiter.recv().await {
|
||||
Ok(req) => break req,
|
||||
Err(broadcast::error::RecvError::Lagged(skipped)) => {
|
||||
tracing::warn!(
|
||||
skipped,
|
||||
"heartbeat reconcile worker lagged, waiting for latest request"
|
||||
);
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => {
|
||||
tracing::error!("Failed to receive heartbeat request: channel closed");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Drop any heartbeat backlog accumulated while the previous reconcile
|
||||
// round was doing DB/RPC IO. The newest heartbeat has the freshest
|
||||
// runtime instance list, which is all this task needs.
|
||||
loop {
|
||||
match heartbeat_waiter.try_recv() {
|
||||
Ok(next_req) => req = next_req,
|
||||
Err(broadcast::error::TryRecvError::Empty) => break,
|
||||
Err(broadcast::error::TryRecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::TryRecvError::Closed) => return None,
|
||||
}
|
||||
}
|
||||
|
||||
Some(req)
|
||||
}
|
||||
|
||||
pub(super) async fn reconcile_network_configs_on_heartbeat(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
mut heartbeat_waiter: broadcast::Receiver<HeartbeatRequest>,
|
||||
storage: WeakRefStorage,
|
||||
mut rpc_client: SessionRpcClient,
|
||||
mut config_client: SessionConfigClient,
|
||||
) {
|
||||
let mut cache = ReconcileCache::default();
|
||||
loop {
|
||||
let Some(req) = recv_latest_heartbeat(&mut heartbeat_waiter).await else {
|
||||
return;
|
||||
};
|
||||
let Some(storage) = storage.upgrade() else {
|
||||
tracing::error!("Failed to get storage");
|
||||
return;
|
||||
};
|
||||
|
||||
let mut round =
|
||||
match prepare_reconcile_round(&session_data, &storage, &mut rpc_client, req).await {
|
||||
RoundStatus::Ready(round) => round,
|
||||
RoundStatus::Skip => continue,
|
||||
RoundStatus::Stop => return,
|
||||
};
|
||||
let running_metas =
|
||||
match sync_running_sources_for_round(&mut rpc_client, &storage, &mut round).await {
|
||||
RoundStatus::Ready(running_metas) => running_metas,
|
||||
RoundStatus::Skip => continue,
|
||||
RoundStatus::Stop => return,
|
||||
};
|
||||
|
||||
let desired_web_inst_ids =
|
||||
managed_config::desired_web_source_instance_ids(&round.local_configs);
|
||||
cache.runtime_configs.retain_desired(&desired_web_inst_ids);
|
||||
let mut outcome = match cleanup_stale_web_source_instances(
|
||||
&session_data,
|
||||
&storage,
|
||||
&mut rpc_client,
|
||||
&round,
|
||||
running_metas.as_deref(),
|
||||
&desired_web_inst_ids,
|
||||
&mut cache,
|
||||
)
|
||||
.await
|
||||
{
|
||||
RoundStatus::Ready(outcome) => outcome,
|
||||
RoundStatus::Skip => continue,
|
||||
RoundStatus::Stop => return,
|
||||
};
|
||||
|
||||
outcome.merge(
|
||||
reconcile_desired_runtime_configs(
|
||||
&session_data,
|
||||
&mut rpc_client,
|
||||
&mut config_client,
|
||||
&round,
|
||||
&mut cache,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
|
||||
if !outcome.has_failed {
|
||||
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids);
|
||||
}
|
||||
match mark_config_revision_applied_if_current(&session_data, &storage, &round, &outcome)
|
||||
.await
|
||||
{
|
||||
RoundStatus::Ready(()) | RoundStatus::Skip => {}
|
||||
RoundStatus::Stop => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum RoundStatus<T> {
|
||||
Ready(T),
|
||||
Skip,
|
||||
Stop,
|
||||
}
|
||||
|
||||
enum ConfigActionResult {
|
||||
Success,
|
||||
Failed,
|
||||
StopRound,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReconcileCache {
|
||||
cleaned_web_source_instances: bool,
|
||||
last_desired_web_inst_ids: Option<HashSet<String>>,
|
||||
runtime_configs: SessionRuntimeConfigCache,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SessionRuntimeConfigCache {
|
||||
entries: HashMap<String, NetworkConfig>,
|
||||
}
|
||||
|
||||
impl SessionRuntimeConfigCache {
|
||||
fn plan(
|
||||
&self,
|
||||
inst_id: &str,
|
||||
desired_config: NetworkConfig,
|
||||
) -> anyhow::Result<Option<runtime_reconcile::RuntimeReconcileAction>> {
|
||||
let Some(observed_config) = self.entries.get(inst_id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
|
||||
observed_config,
|
||||
desired_config,
|
||||
)
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
fn remember(&mut self, inst_id: &str, observed_config: NetworkConfig) {
|
||||
self.entries.insert(inst_id.to_string(), observed_config);
|
||||
}
|
||||
|
||||
fn forget(&mut self, inst_id: &str) {
|
||||
self.entries.remove(inst_id);
|
||||
}
|
||||
|
||||
fn forget_many<'a>(&mut self, inst_ids: impl IntoIterator<Item = &'a String>) {
|
||||
for inst_id in inst_ids {
|
||||
self.entries.remove(inst_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn retain_desired(&mut self, desired_web_inst_ids: &HashSet<String>) {
|
||||
self.entries
|
||||
.retain(|inst_id, _| desired_web_inst_ids.contains(inst_id));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReconcileOutcome {
|
||||
has_failed: bool,
|
||||
managed_revision_failed: bool,
|
||||
}
|
||||
|
||||
impl ReconcileOutcome {
|
||||
fn record_failure(&mut self, managed_revision_failed: bool) {
|
||||
self.has_failed = true;
|
||||
self.managed_revision_failed |= managed_revision_failed;
|
||||
}
|
||||
|
||||
fn merge(&mut self, other: Self) {
|
||||
self.has_failed |= other.has_failed;
|
||||
self.managed_revision_failed |= other.managed_revision_failed;
|
||||
}
|
||||
}
|
||||
|
||||
struct ReconcileRound {
|
||||
req: HeartbeatRequest,
|
||||
machine_id: uuid::Uuid,
|
||||
user_id: i32,
|
||||
running_inst_ids: HashSet<String>,
|
||||
local_configs: Vec<crate::db::entity::user_running_network_configs::Model>,
|
||||
target_config_revision: Option<String>,
|
||||
should_apply_runtime_revision: bool,
|
||||
}
|
||||
|
||||
async fn prepare_reconcile_round(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
storage: &StorageInner,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
req: HeartbeatRequest,
|
||||
) -> RoundStatus<ReconcileRound> {
|
||||
let Some(machine_id) = req.machine_id.map(uuid::Uuid::from) else {
|
||||
tracing::warn!(?req, "Machine id is not set, ignore");
|
||||
return RoundStatus::Skip;
|
||||
};
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &req).await {
|
||||
tracing::debug!(?machine_id, "skip stale heartbeat reconcile request");
|
||||
return RoundStatus::Skip;
|
||||
}
|
||||
|
||||
let user_id = match storage
|
||||
.db
|
||||
.get_user_id_by_token(req.user_token.clone())
|
||||
.await
|
||||
{
|
||||
Ok(Some(user_id)) => user_id,
|
||||
Ok(None) => {
|
||||
tracing::info!("User not found by token: {:?}", req.user_token);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to get user id by token, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
|
||||
let applied_config_revision = {
|
||||
let Some(data) = session_data.upgrade() else {
|
||||
return RoundStatus::Stop;
|
||||
};
|
||||
data.read().await.applied_config_revision.clone()
|
||||
};
|
||||
let target_config_revision = match storage
|
||||
.db
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
{
|
||||
Ok(revision) => revision,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to read managed config revision, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
let should_apply_runtime_revision =
|
||||
target_config_revision.is_some() && target_config_revision != applied_config_revision;
|
||||
let running_inst_ids = match running_instance_ids_for_round(
|
||||
rpc_client,
|
||||
&req,
|
||||
user_id,
|
||||
machine_id,
|
||||
should_apply_runtime_revision,
|
||||
)
|
||||
.await
|
||||
{
|
||||
RoundStatus::Ready(ids) => ids,
|
||||
RoundStatus::Skip => return RoundStatus::Skip,
|
||||
RoundStatus::Stop => return RoundStatus::Stop,
|
||||
};
|
||||
|
||||
let local_configs = match storage
|
||||
.db
|
||||
.list_network_configs((user_id, machine_id), ListNetworkProps::EnabledOnly)
|
||||
.await
|
||||
{
|
||||
Ok(configs) => configs,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to list network configs, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
|
||||
RoundStatus::Ready(ReconcileRound {
|
||||
req,
|
||||
machine_id,
|
||||
user_id,
|
||||
running_inst_ids,
|
||||
local_configs,
|
||||
target_config_revision,
|
||||
should_apply_runtime_revision,
|
||||
})
|
||||
}
|
||||
|
||||
async fn running_instance_ids_for_round(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
req: &HeartbeatRequest,
|
||||
user_id: i32,
|
||||
machine_id: uuid::Uuid,
|
||||
should_apply_runtime_revision: bool,
|
||||
) -> RoundStatus<HashSet<String>> {
|
||||
if !should_apply_runtime_revision {
|
||||
return RoundStatus::Ready(
|
||||
req.running_network_instances
|
||||
.iter()
|
||||
.map(|x| x.to_string())
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
|
||||
match rpc_client
|
||||
.list_network_instance(BaseController::default(), ListNetworkInstanceRequest {})
|
||||
.await
|
||||
{
|
||||
Ok(resp) => RoundStatus::Ready(resp.inst_ids.iter().map(|x| x.to_string()).collect()),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
?user_id,
|
||||
?machine_id,
|
||||
?error,
|
||||
"Failed to refresh running instances for managed config revision"
|
||||
);
|
||||
RoundStatus::Skip
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn sync_running_sources_for_round(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
storage: &StorageInner,
|
||||
round: &mut ReconcileRound,
|
||||
) -> RoundStatus<Option<Vec<NetworkMeta>>> {
|
||||
if !round.req.support_config_source {
|
||||
return RoundStatus::Ready(None);
|
||||
}
|
||||
|
||||
let ret = if round.running_inst_ids.is_empty() {
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
rpc_client
|
||||
.list_network_instance_meta(
|
||||
BaseController::default(),
|
||||
ListNetworkInstanceMetaRequest {
|
||||
inst_ids: managed_config::parse_instance_ids(
|
||||
round.running_inst_ids.iter().cloned(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|resp| resp.metas)
|
||||
};
|
||||
|
||||
match ret {
|
||||
Ok(metas) => {
|
||||
if let Err(e) = managed_config::sync_running_config_sources(
|
||||
&storage.db,
|
||||
round.user_id,
|
||||
round.machine_id,
|
||||
&round.local_configs,
|
||||
&metas,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
user_id = ?round.user_id,
|
||||
machine_id = ?round.machine_id,
|
||||
%e,
|
||||
"Failed to sync running network config sources"
|
||||
);
|
||||
} else if !metas.is_empty() {
|
||||
round.local_configs = match storage
|
||||
.db
|
||||
.list_network_configs(
|
||||
(round.user_id, round.machine_id),
|
||||
ListNetworkProps::EnabledOnly,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(configs) => configs,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"Failed to reload network configs after source sync, error: {:?}",
|
||||
e
|
||||
);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
}
|
||||
RoundStatus::Ready(Some(metas))
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
user_id = ?round.user_id,
|
||||
%e,
|
||||
"Failed to list running network instance metadata"
|
||||
);
|
||||
RoundStatus::Ready(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_stale_web_source_instances(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
storage: &StorageInner,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
round: &ReconcileRound,
|
||||
running_metas: Option<&[NetworkMeta]>,
|
||||
desired_web_inst_ids: &HashSet<String>,
|
||||
cache: &mut ReconcileCache,
|
||||
) -> RoundStatus<ReconcileOutcome> {
|
||||
let desired_changed = cache
|
||||
.last_desired_web_inst_ids
|
||||
.as_ref()
|
||||
.is_none_or(|last| last != desired_web_inst_ids);
|
||||
if cache.cleaned_web_source_instances && !desired_changed {
|
||||
return RoundStatus::Ready(ReconcileOutcome::default());
|
||||
}
|
||||
|
||||
let db_web_inst_ids = match storage
|
||||
.db
|
||||
.list_network_configs((round.user_id, round.machine_id), ListNetworkProps::All)
|
||||
.await
|
||||
{
|
||||
Ok(configs) => managed_config::desired_web_source_instance_ids(&configs),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to list all network configs, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
|
||||
let running_web_inst_ids = managed_config::running_web_source_instance_ids(
|
||||
&round.running_inst_ids,
|
||||
&db_web_inst_ids,
|
||||
running_metas,
|
||||
);
|
||||
let should_delete_inst_ids = running_web_inst_ids
|
||||
.difference(desired_web_inst_ids)
|
||||
.cloned()
|
||||
.collect::<HashSet<_>>();
|
||||
let should_delete_ids =
|
||||
managed_config::parse_instance_ids(should_delete_inst_ids.iter().cloned());
|
||||
|
||||
let mut outcome = ReconcileOutcome::default();
|
||||
if !should_delete_ids.is_empty() {
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
tracing::debug!(
|
||||
machine_id = ?round.machine_id,
|
||||
"skip stale cleanup because webhook session is no longer current"
|
||||
);
|
||||
return RoundStatus::Skip;
|
||||
}
|
||||
let ret = rpc_client
|
||||
.delete_network_instance(
|
||||
BaseController::default(),
|
||||
DeleteNetworkInstanceRequest {
|
||||
inst_ids: should_delete_ids,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
user_id = ?round.user_id,
|
||||
"Clean stale web-source network instances on heartbeat: {:?}, user_token: {:?}",
|
||||
ret,
|
||||
round.req.user_token
|
||||
);
|
||||
if ret.is_err() {
|
||||
outcome.record_failure(true);
|
||||
} else {
|
||||
cache.runtime_configs.forget_many(&should_delete_inst_ids);
|
||||
}
|
||||
}
|
||||
|
||||
if !outcome.has_failed {
|
||||
cache.cleaned_web_source_instances = true;
|
||||
cache.last_desired_web_inst_ids = Some(desired_web_inst_ids.clone());
|
||||
}
|
||||
|
||||
RoundStatus::Ready(outcome)
|
||||
}
|
||||
|
||||
async fn reconcile_desired_runtime_configs(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
config_client: &mut SessionConfigClient,
|
||||
round: &ReconcileRound,
|
||||
cache: &mut ReconcileCache,
|
||||
) -> ReconcileOutcome {
|
||||
let mut outcome = ReconcileOutcome::default();
|
||||
|
||||
// After stale web-owned instances are removed, start every enabled
|
||||
// config that the latest heartbeat did not report as running. When
|
||||
// a managed config revision is pending, also reconcile running
|
||||
// web-owned configs before reporting that revision as applied.
|
||||
for config in &round.local_configs {
|
||||
let source = PersistedConfigSource::from_db(&config.source);
|
||||
let is_running = round.running_inst_ids.contains(&config.network_instance_id);
|
||||
let should_reconcile_running_web_config = is_running
|
||||
&& round.should_apply_runtime_revision
|
||||
&& source == PersistedConfigSource::Web;
|
||||
if is_running && !should_reconcile_running_web_config {
|
||||
continue;
|
||||
}
|
||||
|
||||
let desired_config = match serde_json::from_str::<NetworkConfig>(&config.network_config) {
|
||||
Ok(cfg) => cfg,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
user_id = ?round.user_id,
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"Failed to deserialize network config, skipping: {:?}",
|
||||
e
|
||||
);
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
outcome.record_failure(source == PersistedConfigSource::Web);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let action_result = if should_reconcile_running_web_config {
|
||||
reconcile_running_web_config(
|
||||
session_data,
|
||||
rpc_client,
|
||||
config_client,
|
||||
round,
|
||||
config,
|
||||
desired_config,
|
||||
&mut cache.runtime_configs,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
let action_result = run_missing_network_config(
|
||||
session_data,
|
||||
rpc_client,
|
||||
round,
|
||||
config,
|
||||
desired_config.clone(),
|
||||
)
|
||||
.await;
|
||||
if matches!(action_result, ConfigActionResult::Success)
|
||||
&& source == PersistedConfigSource::Web
|
||||
{
|
||||
if let Err(e) = remember_web_runtime_config_after_run(
|
||||
rpc_client,
|
||||
&config.network_instance_id,
|
||||
&desired_config,
|
||||
&mut cache.runtime_configs,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!(
|
||||
user_id = ?round.user_id,
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"Failed to cache runtime config after run: {:?}",
|
||||
e
|
||||
);
|
||||
ConfigActionResult::Failed
|
||||
} else {
|
||||
action_result
|
||||
}
|
||||
} else {
|
||||
action_result
|
||||
}
|
||||
};
|
||||
|
||||
match action_result {
|
||||
ConfigActionResult::Success => {}
|
||||
ConfigActionResult::Failed => {
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
outcome.record_failure(source == PersistedConfigSource::Web)
|
||||
}
|
||||
ConfigActionResult::StopRound => {
|
||||
if source == PersistedConfigSource::Web {
|
||||
cache.runtime_configs.forget(&config.network_instance_id);
|
||||
}
|
||||
outcome.record_failure(source == PersistedConfigSource::Web);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
outcome
|
||||
}
|
||||
|
||||
async fn reconcile_running_web_config(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
config_client: &mut SessionConfigClient,
|
||||
round: &ReconcileRound,
|
||||
config: &crate::db::entity::user_running_network_configs::Model,
|
||||
desired_config: NetworkConfig,
|
||||
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||
) -> ConfigActionResult {
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
tracing::debug!(
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"skip runtime reconcile because webhook session is no longer current"
|
||||
);
|
||||
return ConfigActionResult::StopRound;
|
||||
}
|
||||
|
||||
let ret = async {
|
||||
let action =
|
||||
match runtime_config_cache.plan(&config.network_instance_id, desired_config.clone())? {
|
||||
Some(action) => action,
|
||||
None => {
|
||||
runtime_reconcile::prepare_web_source_runtime_reconcile(
|
||||
&mut *rpc_client,
|
||||
&config.network_instance_id,
|
||||
desired_config.clone(),
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
anyhow::bail!("webhook session is no longer current before runtime reconcile apply");
|
||||
}
|
||||
let observed_config = runtime_reconcile::apply_web_source_runtime_reconcile(
|
||||
&mut *rpc_client,
|
||||
&mut *config_client,
|
||||
&config.network_instance_id,
|
||||
desired_config.clone(),
|
||||
action,
|
||||
)
|
||||
.await?;
|
||||
runtime_config_cache.remember(&config.network_instance_id, observed_config);
|
||||
Ok::<(), anyhow::Error>(())
|
||||
}
|
||||
.await;
|
||||
tracing::info!(
|
||||
user_id = ?round.user_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"Reconcile running web-source network instance: {:?}, user_token: {:?}",
|
||||
ret,
|
||||
round.req.user_token
|
||||
);
|
||||
|
||||
if ret.is_ok() {
|
||||
ConfigActionResult::Success
|
||||
} else {
|
||||
runtime_config_cache.forget(&config.network_instance_id);
|
||||
ConfigActionResult::Failed
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_missing_network_config(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
round: &ReconcileRound,
|
||||
config: &crate::db::entity::user_running_network_configs::Model,
|
||||
desired_config: NetworkConfig,
|
||||
) -> ConfigActionResult {
|
||||
if !SessionRpcService::runtime_heartbeat_is_current(session_data, &round.req).await {
|
||||
tracing::debug!(
|
||||
machine_id = ?round.machine_id,
|
||||
instance_id = %config.network_instance_id,
|
||||
"skip run network instance because webhook session is no longer current"
|
||||
);
|
||||
return ConfigActionResult::StopRound;
|
||||
}
|
||||
|
||||
let ret = rpc_client
|
||||
.run_network_instance(
|
||||
BaseController::default(),
|
||||
RunNetworkInstanceRequest {
|
||||
inst_id: Some(config.network_instance_id.clone().into()),
|
||||
config: Some(desired_config),
|
||||
overwrite: false,
|
||||
source: PersistedConfigSource::from_db(&config.source).auto_run_rpc_source() as i32,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
user_id = ?round.user_id,
|
||||
"Run network instance: {:?}, user_token: {:?}",
|
||||
ret,
|
||||
round.req.user_token
|
||||
);
|
||||
|
||||
if ret.is_ok() {
|
||||
ConfigActionResult::Success
|
||||
} else {
|
||||
ConfigActionResult::Failed
|
||||
}
|
||||
}
|
||||
|
||||
async fn remember_web_runtime_config_after_run(
|
||||
rpc_client: &mut SessionRpcClient,
|
||||
inst_id: &str,
|
||||
desired_config: &NetworkConfig,
|
||||
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||
) -> anyhow::Result<()> {
|
||||
let observed_config = runtime_reconcile::get_runtime_config(rpc_client, inst_id).await?;
|
||||
remember_if_runtime_matches_desired(
|
||||
inst_id,
|
||||
desired_config,
|
||||
observed_config,
|
||||
runtime_config_cache,
|
||||
)
|
||||
}
|
||||
|
||||
fn remember_if_runtime_matches_desired(
|
||||
inst_id: &str,
|
||||
desired_config: &NetworkConfig,
|
||||
observed_config: NetworkConfig,
|
||||
runtime_config_cache: &mut SessionRuntimeConfigCache,
|
||||
) -> anyhow::Result<()> {
|
||||
let action = runtime_reconcile::prepare_web_source_runtime_reconcile_from_current(
|
||||
&observed_config,
|
||||
desired_config.clone(),
|
||||
)?;
|
||||
if !matches!(action, runtime_reconcile::RuntimeReconcileAction::None) {
|
||||
anyhow::bail!("runtime config still differs after managed run");
|
||||
}
|
||||
runtime_config_cache.remember(inst_id, observed_config);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_config_revision_applied_if_current(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
storage: &StorageInner,
|
||||
round: &ReconcileRound,
|
||||
outcome: &ReconcileOutcome,
|
||||
) -> RoundStatus<()> {
|
||||
if outcome.managed_revision_failed || !round.should_apply_runtime_revision {
|
||||
return RoundStatus::Ready(());
|
||||
}
|
||||
|
||||
let current_target_config_revision = match storage
|
||||
.db
|
||||
.get_managed_config_revision((round.user_id, round.machine_id))
|
||||
.await
|
||||
{
|
||||
Ok(revision) => revision,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to verify managed config revision, error: {:?}", e);
|
||||
return RoundStatus::Stop;
|
||||
}
|
||||
};
|
||||
if current_target_config_revision != round.target_config_revision {
|
||||
return RoundStatus::Ready(());
|
||||
}
|
||||
let Some(data) = session_data.upgrade() else {
|
||||
return RoundStatus::Stop;
|
||||
};
|
||||
let mut data = data.write().await;
|
||||
if !SessionRpcService::runtime_heartbeat_is_current_locked(&data, &round.req) {
|
||||
return RoundStatus::Ready(());
|
||||
}
|
||||
data.applied_config_revision = round.target_config_revision.clone();
|
||||
|
||||
RoundStatus::Ready(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use easytier::proto::api::manage::{NetworkingMethod, PortForwardConfig};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn config_with_port_forwards(port_forwards: Vec<PortForwardConfig>) -> NetworkConfig {
|
||||
NetworkConfig {
|
||||
instance_id: Some("11111111-1111-1111-1111-111111111111".to_string()),
|
||||
dhcp: Some(true),
|
||||
network_name: Some("managed".to_string()),
|
||||
network_secret: Some("secret".to_string()),
|
||||
networking_method: Some(NetworkingMethod::Manual as i32),
|
||||
port_forwards,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn port_forward(bind_port: u32, dst_port: u32) -> PortForwardConfig {
|
||||
PortForwardConfig {
|
||||
bind_ip: "127.0.0.1".to_string(),
|
||||
bind_port,
|
||||
dst_ip: "10.144.0.1".to_string(),
|
||||
dst_port,
|
||||
proto: "tcp".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_misses_unknown_instance() {
|
||||
let cache = SessionRuntimeConfigCache::default();
|
||||
let action = cache
|
||||
.plan("missing", config_with_port_forwards(Vec::new()))
|
||||
.expect("prepare action");
|
||||
|
||||
assert!(action.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_skips_matching_observed_config() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
cache.remember("managed", config.clone());
|
||||
let action = cache
|
||||
.plan("managed", config)
|
||||
.expect("prepare action")
|
||||
.expect("cached action");
|
||||
|
||||
assert!(matches!(
|
||||
action,
|
||||
runtime_reconcile::RuntimeReconcileAction::None
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_plans_patch_from_observed_config() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
cache.remember("managed", current);
|
||||
let action = cache
|
||||
.plan("managed", desired)
|
||||
.expect("prepare action")
|
||||
.expect("cached action");
|
||||
|
||||
let runtime_reconcile::RuntimeReconcileAction::Patch(patch) = action else {
|
||||
panic!("expected cached runtime config to produce hot patch");
|
||||
};
|
||||
assert_eq!(patch.port_forwards.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_retain_desired_removes_stale_entries() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(Vec::new());
|
||||
cache.remember("keep", config.clone());
|
||||
cache.remember("drop", config);
|
||||
|
||||
cache.retain_desired(&HashSet::from(["keep".to_string()]));
|
||||
|
||||
assert!(cache.entries.contains_key("keep"));
|
||||
assert!(!cache.entries.contains_key("drop"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_runtime_config_cache_forget_removes_observed_config() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(Vec::new());
|
||||
cache.remember("managed", config.clone());
|
||||
|
||||
cache.forget("managed");
|
||||
|
||||
let action = cache
|
||||
.plan("managed", config)
|
||||
.expect("prepare action after remove");
|
||||
assert!(action.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_run_remembers_observed_config_when_it_matches_desired() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let config = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
|
||||
remember_if_runtime_matches_desired("managed", &config, config.clone(), &mut cache)
|
||||
.expect("remember observed config after run");
|
||||
let action = cache
|
||||
.plan("managed", config)
|
||||
.expect("prepare action after run")
|
||||
.expect("cached action");
|
||||
|
||||
assert!(matches!(
|
||||
action,
|
||||
runtime_reconcile::RuntimeReconcileAction::None
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_run_does_not_remember_observed_config_that_still_differs() {
|
||||
let mut cache = SessionRuntimeConfigCache::default();
|
||||
let current = config_with_port_forwards(vec![port_forward(23000, 5174)]);
|
||||
let desired =
|
||||
config_with_port_forwards(vec![port_forward(23000, 5174), port_forward(23007, 3389)]);
|
||||
|
||||
let err = remember_if_runtime_matches_desired("managed", &desired, current, &mut cache)
|
||||
.expect_err("expected stale run result not to be cached");
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("runtime config still differs after managed run")
|
||||
);
|
||||
let action = cache
|
||||
.plan("managed", desired)
|
||||
.expect("prepare action after stale run result");
|
||||
assert!(action.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,402 +0,0 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use anyhow::Context as _;
|
||||
use easytier::proto::web::HeartbeatRequest;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::{
|
||||
SessionAuthState, SessionData, SessionRpcService, WebhookConnectNotification,
|
||||
WebhookDisconnectNotification, send_webhook_connection_transition,
|
||||
};
|
||||
use crate::{
|
||||
client_manager::storage::{Storage, StorageToken},
|
||||
webhook::SharedWebhookConfig,
|
||||
};
|
||||
|
||||
pub(super) const VALIDATION_RETRY_MS: u64 = 60_000;
|
||||
|
||||
pub(super) struct WebhookHeartbeatValidation {
|
||||
pub(super) config_revision: String,
|
||||
pub(super) binding_version: u64,
|
||||
}
|
||||
|
||||
pub(super) struct WebhookValidationInput {
|
||||
pub(super) storage: Storage,
|
||||
pub(super) webhook_config: SharedWebhookConfig,
|
||||
pub(super) client_url: url::Url,
|
||||
pub(super) applied_config_revision: Option<String>,
|
||||
pub(super) req: HeartbeatRequest,
|
||||
pub(super) machine_id: uuid::Uuid,
|
||||
}
|
||||
|
||||
fn deterministic_machine_delay(machine_id: uuid::Uuid, max_delay_ms: u64) -> Duration {
|
||||
let delay_ms = (machine_id.as_u128() % u128::from(max_delay_ms + 1)) as u64;
|
||||
Duration::from_millis(delay_ms)
|
||||
}
|
||||
|
||||
pub(super) fn retry_delay(machine_id: uuid::Uuid) -> Duration {
|
||||
Duration::from_millis(VALIDATION_RETRY_MS)
|
||||
+ deterministic_machine_delay(machine_id, VALIDATION_RETRY_MS)
|
||||
}
|
||||
|
||||
async fn request_heartbeat_validation(
|
||||
webhook_config: &crate::webhook::WebhookConfig,
|
||||
client_url: &url::Url,
|
||||
persisted_config_revision: Option<&str>,
|
||||
applied_config_revision: Option<&str>,
|
||||
req: &HeartbeatRequest,
|
||||
machine_id: uuid::Uuid,
|
||||
) -> anyhow::Result<Option<WebhookHeartbeatValidation>> {
|
||||
let webhook_req = crate::webhook::ValidateTokenRequest {
|
||||
token: req.user_token.clone(),
|
||||
machine_id: machine_id.to_string(),
|
||||
public_ip: client_url.host_str().map(str::to_string),
|
||||
hostname: req.hostname.clone(),
|
||||
version: req.easytier_version.clone(),
|
||||
os_type: req.device_os.as_ref().map(|info| info.os_type.clone()),
|
||||
os_version: req.device_os.as_ref().map(|info| info.version.clone()),
|
||||
os_distribution: req.device_os.as_ref().map(|info| info.distribution.clone()),
|
||||
web_instance_id: webhook_config.web_instance_id.clone(),
|
||||
web_instance_api_base_url: webhook_config.web_instance_api_base_url.clone(),
|
||||
persisted_config_revision: persisted_config_revision.map(str::to_string),
|
||||
applied_config_revision: applied_config_revision.map(str::to_string),
|
||||
};
|
||||
let resp = webhook_config
|
||||
.validate_token(&webhook_req)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Webhook token validation failed: {:?}", e))?;
|
||||
|
||||
if !resp.valid {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(Some(WebhookHeartbeatValidation {
|
||||
config_revision: resp.config_revision,
|
||||
binding_version: resp.binding_version,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn resolve_user_id(storage: &Storage, token: &str) -> anyhow::Result<i32> {
|
||||
let user_id = match storage
|
||||
.db()
|
||||
.get_user_id_by_token(token)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
|
||||
{
|
||||
Some(id) => id,
|
||||
None => storage
|
||||
.auto_create_user(token)
|
||||
.await
|
||||
.with_context(|| format!("Failed to auto-create webhook user: {:?}", token))?,
|
||||
};
|
||||
|
||||
Ok(user_id)
|
||||
}
|
||||
|
||||
async fn persisted_config_revision_for_token(
|
||||
storage: &Storage,
|
||||
token: &str,
|
||||
machine_id: uuid::Uuid,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
let Some(user_id) = storage
|
||||
.db()
|
||||
.get_user_id_by_token(token)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
storage
|
||||
.db()
|
||||
.get_managed_config_revision((user_id, machine_id))
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("DB error: {:?}", e))
|
||||
}
|
||||
|
||||
async fn wait_for_input(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
) -> Option<WebhookValidationInput> {
|
||||
loop {
|
||||
let notify = {
|
||||
let session_data = session_data.upgrade()?;
|
||||
let mut data = session_data.write().await;
|
||||
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||
data.webhook_validation_dirty = false;
|
||||
tracing::info!(
|
||||
client_url = %data.client_url,
|
||||
"webhook validation stopped for invalid session; reconnect is required before revalidation"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
if data.webhook_validation_dirty {
|
||||
data.webhook_validation_dirty = false;
|
||||
let req = data.req.clone()?;
|
||||
let machine_id = req.machine_id.map(Into::into)?;
|
||||
let storage = Storage::try_from(data.storage.clone()).ok()?;
|
||||
return Some(WebhookValidationInput {
|
||||
storage,
|
||||
webhook_config: data.webhook_config.clone(),
|
||||
client_url: data.client_url.clone(),
|
||||
applied_config_revision: data.applied_config_revision.clone(),
|
||||
req,
|
||||
machine_id,
|
||||
});
|
||||
}
|
||||
data.webhook_validation_notify.clone()
|
||||
};
|
||||
notify.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn run_worker(session_data: std::sync::Weak<RwLock<SessionData>>) {
|
||||
while let Some(input) = wait_for_input(session_data.clone()).await {
|
||||
let machine_id = input.machine_id;
|
||||
if let Err(error) = run_round(session_data.clone(), input).await {
|
||||
tracing::warn!(
|
||||
?machine_id,
|
||||
%error,
|
||||
"webhook validation failed, will retry later"
|
||||
);
|
||||
tokio::time::sleep(retry_delay(machine_id)).await;
|
||||
mark_dirty_if_current(&session_data, machine_id).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn run_round(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
input: WebhookValidationInput,
|
||||
) -> anyhow::Result<()> {
|
||||
let persisted_config_revision = persisted_config_revision_for_token(
|
||||
&input.storage,
|
||||
&input.req.user_token,
|
||||
input.machine_id,
|
||||
)
|
||||
.await?;
|
||||
let validation = request_heartbeat_validation(
|
||||
&input.webhook_config,
|
||||
&input.client_url,
|
||||
persisted_config_revision.as_deref(),
|
||||
input.applied_config_revision.as_deref(),
|
||||
&input.req,
|
||||
input.machine_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let Some(validation) = validation else {
|
||||
apply_rejected(&session_data, &input).await;
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let user_id = resolve_user_id(&input.storage, &input.req.user_token).await?;
|
||||
apply_success(&session_data, input, validation, user_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_dirty_if_current(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
machine_id: uuid::Uuid,
|
||||
) {
|
||||
let Some(session_data) = session_data.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let notify = {
|
||||
let mut data = session_data.write().await;
|
||||
let Some(req) = data.req.as_ref() else {
|
||||
return;
|
||||
};
|
||||
if req.machine_id.map(uuid::Uuid::from) != Some(machine_id) {
|
||||
return;
|
||||
}
|
||||
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||
data.webhook_validation_dirty = false;
|
||||
tracing::debug!(
|
||||
%machine_id,
|
||||
"skip webhook validation retry for invalid session"
|
||||
);
|
||||
return;
|
||||
}
|
||||
SessionRpcService::mark_webhook_validation_dirty_locked(&mut data)
|
||||
};
|
||||
notify.notify_one();
|
||||
}
|
||||
|
||||
pub(super) async fn apply_rejected(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
input: &WebhookValidationInput,
|
||||
) {
|
||||
let Some(session_data) = session_data.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let (storage_token, disconnect_notification) = {
|
||||
let mut data = session_data.write().await;
|
||||
if !data.req.as_ref().is_some_and(|req| {
|
||||
SessionRpcService::heartbeat_matches_identity(
|
||||
req,
|
||||
&input.req.user_token,
|
||||
input.machine_id,
|
||||
)
|
||||
}) {
|
||||
return;
|
||||
}
|
||||
tracing::info!(
|
||||
machine_id = %input.machine_id,
|
||||
client_url = %data.client_url,
|
||||
"webhook token rejected; marking session invalid and requiring client reconnect"
|
||||
);
|
||||
data.auth_state = SessionAuthState::Invalid;
|
||||
data.webhook_validation_dirty = false;
|
||||
data.binding_version = None;
|
||||
data.applied_config_revision = None;
|
||||
let storage_token = data.storage_token.clone();
|
||||
let disconnect_notification = storage_token.as_ref().and_then(|storage_token| {
|
||||
data.webhook_connected_binding_version
|
||||
.take()
|
||||
.map(|binding_version| WebhookDisconnectNotification {
|
||||
webhook: data.webhook_config.clone(),
|
||||
storage_token: storage_token.clone(),
|
||||
binding_version,
|
||||
})
|
||||
});
|
||||
(storage_token, disconnect_notification)
|
||||
};
|
||||
if let Some(storage_token) = storage_token {
|
||||
let report_time = SessionRpcService::heartbeat_report_timestamp(&input.req);
|
||||
input
|
||||
.storage
|
||||
.update_client(storage_token, report_time, false);
|
||||
}
|
||||
if disconnect_notification.is_some() {
|
||||
wait_webhook_connection_transition(
|
||||
Arc::downgrade(&session_data),
|
||||
disconnect_notification,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn apply_success(
|
||||
session_data: &std::sync::Weak<RwLock<SessionData>>,
|
||||
input: WebhookValidationInput,
|
||||
validation: WebhookHeartbeatValidation,
|
||||
user_id: i32,
|
||||
) {
|
||||
let WebhookHeartbeatValidation {
|
||||
config_revision: _,
|
||||
binding_version,
|
||||
} = validation;
|
||||
|
||||
let Some(session_data) = session_data.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let (storage_token, notifier, disconnect_notification, connect_notification, runtime_req) = {
|
||||
let mut data = session_data.write().await;
|
||||
let Some(runtime_req) = data.req.clone() else {
|
||||
return;
|
||||
};
|
||||
if !SessionRpcService::heartbeat_matches_identity(
|
||||
&runtime_req,
|
||||
&input.req.user_token,
|
||||
input.machine_id,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if matches!(data.auth_state, SessionAuthState::Invalid) {
|
||||
tracing::info!(
|
||||
machine_id = %input.machine_id,
|
||||
client_url = %data.client_url,
|
||||
"ignore webhook validation success for invalid session; reconnect is required before revalidation"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let previous_connected_binding_version = data.webhook_connected_binding_version;
|
||||
let client_url = data.client_url.clone();
|
||||
let storage_token = data.storage_token.get_or_insert_with(|| StorageToken {
|
||||
token: runtime_req.user_token.clone(),
|
||||
client_url,
|
||||
machine_id: input.machine_id,
|
||||
user_id,
|
||||
});
|
||||
let storage_token = storage_token.clone();
|
||||
data.auth_state = SessionAuthState::Authorized;
|
||||
data.binding_version = Some(binding_version);
|
||||
let should_notify_connected = previous_connected_binding_version != Some(binding_version);
|
||||
let disconnect_notification = previous_connected_binding_version
|
||||
.filter(|previous_binding_version| *previous_binding_version != binding_version)
|
||||
.map(|previous_binding_version| {
|
||||
data.webhook_connected_binding_version = None;
|
||||
WebhookDisconnectNotification {
|
||||
webhook: data.webhook_config.clone(),
|
||||
storage_token: storage_token.clone(),
|
||||
binding_version: previous_binding_version,
|
||||
}
|
||||
});
|
||||
|
||||
let connect_notification = should_notify_connected.then(|| WebhookConnectNotification {
|
||||
webhook: data.webhook_config.clone(),
|
||||
storage_token: storage_token.clone(),
|
||||
binding_version,
|
||||
req: crate::webhook::NodeConnectedRequest {
|
||||
machine_id: input.machine_id.to_string(),
|
||||
token: runtime_req.user_token.clone(),
|
||||
user_id: Some(user_id),
|
||||
hostname: runtime_req.hostname.clone(),
|
||||
version: runtime_req.easytier_version.clone(),
|
||||
os_type: runtime_req
|
||||
.device_os
|
||||
.as_ref()
|
||||
.map(|info| info.os_type.clone()),
|
||||
os_version: runtime_req
|
||||
.device_os
|
||||
.as_ref()
|
||||
.map(|info| info.version.clone()),
|
||||
os_distribution: runtime_req
|
||||
.device_os
|
||||
.as_ref()
|
||||
.map(|info| info.distribution.clone()),
|
||||
web_instance_id: data.webhook_config.web_instance_id.clone(),
|
||||
binding_version: Some(binding_version),
|
||||
},
|
||||
});
|
||||
|
||||
(
|
||||
storage_token,
|
||||
data.notifier.clone(),
|
||||
disconnect_notification,
|
||||
connect_notification,
|
||||
runtime_req,
|
||||
)
|
||||
};
|
||||
|
||||
if disconnect_notification.is_some() || connect_notification.is_some() {
|
||||
wait_webhook_connection_transition(
|
||||
Arc::downgrade(&session_data),
|
||||
disconnect_notification,
|
||||
connect_notification,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let report_time = SessionRpcService::heartbeat_report_timestamp(&runtime_req);
|
||||
input
|
||||
.storage
|
||||
.update_client(storage_token, report_time, true);
|
||||
let _ = notifier.send(runtime_req);
|
||||
}
|
||||
|
||||
async fn wait_webhook_connection_transition(
|
||||
session_data: std::sync::Weak<RwLock<SessionData>>,
|
||||
disconnect: Option<WebhookDisconnectNotification>,
|
||||
connect: Option<WebhookConnectNotification>,
|
||||
) {
|
||||
let transition = tokio::spawn(send_webhook_connection_transition(
|
||||
session_data,
|
||||
disconnect,
|
||||
connect,
|
||||
));
|
||||
if let Err(error) = transition.await {
|
||||
tracing::warn!(%error, "webhook connection transition task failed");
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ pub struct StorageToken {
|
||||
struct ClientInfo {
|
||||
storage_token: StorageToken,
|
||||
report_time: i64,
|
||||
authorized: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -56,19 +55,7 @@ impl Storage {
|
||||
fn update_client_info_map(map: &DashMap<uuid::Uuid, ClientInfo>, client_info: &ClientInfo) {
|
||||
map.entry(client_info.storage_token.machine_id)
|
||||
.and_modify(|e| {
|
||||
let same_client = e.storage_token.client_url
|
||||
== client_info.storage_token.client_url
|
||||
&& e.storage_token.user_id == client_info.storage_token.user_id;
|
||||
let should_replace = if (same_client && e.authorized != client_info.authorized)
|
||||
|| (!e.authorized && client_info.authorized)
|
||||
{
|
||||
true
|
||||
} else if e.authorized && !client_info.authorized && !same_client {
|
||||
false
|
||||
} else {
|
||||
e.report_time < client_info.report_time
|
||||
};
|
||||
if should_replace {
|
||||
if e.report_time < client_info.report_time {
|
||||
assert_eq!(
|
||||
e.storage_token.machine_id,
|
||||
client_info.storage_token.machine_id
|
||||
@@ -79,13 +66,12 @@ impl Storage {
|
||||
.or_insert(client_info.clone());
|
||||
}
|
||||
|
||||
pub fn update_client(&self, stoken: StorageToken, report_time: i64, authorized: bool) {
|
||||
pub fn update_client(&self, stoken: StorageToken, report_time: i64) {
|
||||
let inner = self.0.user_clients_map.entry(stoken.user_id).or_default();
|
||||
|
||||
let client_info = ClientInfo {
|
||||
storage_token: stoken.clone(),
|
||||
report_time,
|
||||
authorized,
|
||||
};
|
||||
Self::update_client_info_map(&inner, &client_info);
|
||||
}
|
||||
@@ -107,21 +93,11 @@ impl Storage {
|
||||
&self,
|
||||
user_id: UserIdInDb,
|
||||
machine_id: &uuid::Uuid,
|
||||
) -> Option<url::Url> {
|
||||
self.get_client_url_by_machine_id_with_auth(user_id, machine_id, true)
|
||||
}
|
||||
|
||||
pub fn get_client_url_by_machine_id_with_auth(
|
||||
&self,
|
||||
user_id: UserIdInDb,
|
||||
machine_id: &uuid::Uuid,
|
||||
require_authorized: bool,
|
||||
) -> Option<url::Url> {
|
||||
self.0.user_clients_map.get(&user_id).and_then(|info_map| {
|
||||
info_map.get(machine_id).and_then(|info| {
|
||||
(!require_authorized || info.authorized)
|
||||
.then(|| info.storage_token.client_url.clone())
|
||||
})
|
||||
info_map
|
||||
.get(machine_id)
|
||||
.map(|info| info.storage_token.client_url.clone())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -132,7 +108,6 @@ impl Storage {
|
||||
.map(|info_map| {
|
||||
info_map
|
||||
.iter()
|
||||
.filter(|info| info.value().authorized)
|
||||
.map(|info| info.value().storage_token.client_url.clone())
|
||||
.collect()
|
||||
})
|
||||
@@ -140,14 +115,6 @@ impl Storage {
|
||||
}
|
||||
|
||||
pub fn list_clients(&self) -> Vec<StorageToken> {
|
||||
self.list_clients_with_auth(true)
|
||||
}
|
||||
|
||||
pub fn list_all_clients(&self) -> Vec<StorageToken> {
|
||||
self.list_clients_with_auth(false)
|
||||
}
|
||||
|
||||
fn list_clients_with_auth(&self, require_authorized: bool) -> Vec<StorageToken> {
|
||||
self.0
|
||||
.user_clients_map
|
||||
.iter()
|
||||
@@ -155,7 +122,6 @@ impl Storage {
|
||||
user_clients
|
||||
.value()
|
||||
.iter()
|
||||
.filter(|info| !require_authorized || info.value().authorized)
|
||||
.map(|info| info.value().storage_token.clone())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
@@ -198,8 +164,8 @@ mod tests {
|
||||
let user1_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let user2_token = make_storage_token(2, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(user1_token.clone(), 10, true);
|
||||
storage.update_client(user2_token.clone(), 20, true);
|
||||
storage.update_client(user1_token.clone(), 10);
|
||||
storage.update_client(user2_token.clone(), 20);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
@@ -229,8 +195,8 @@ mod tests {
|
||||
let user1_token = make_storage_token(1, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1001");
|
||||
let user2_token = make_storage_token(2, uuid::Uuid::new_v4(), "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(user1_token.clone(), 10, true);
|
||||
storage.update_client(user2_token.clone(), 20, true);
|
||||
storage.update_client(user1_token.clone(), 10);
|
||||
storage.update_client(user2_token.clone(), 20);
|
||||
|
||||
let tokens = storage.list_clients();
|
||||
assert_eq!(tokens.len(), 2);
|
||||
@@ -243,84 +209,4 @@ mod tests {
|
||||
assert_eq!(tokens.len(), 1);
|
||||
assert_eq!(tokens[0].token, user2_token.token);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_client_is_listed_but_not_authorized_for_machine_lookup() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
|
||||
storage.update_client(token.clone(), 10, false);
|
||||
|
||||
assert_eq!(storage.list_clients().len(), 0);
|
||||
assert_eq!(storage.list_all_clients().len(), 1);
|
||||
assert_eq!(storage.list_user_clients(1), Vec::<url::Url>::new());
|
||||
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id_with_auth(1, &machine_id, false),
|
||||
Some(token.client_url.clone())
|
||||
);
|
||||
|
||||
storage.update_client(token.clone(), 11, true);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(token.client_url.clone())
|
||||
);
|
||||
|
||||
storage.update_client(token.clone(), 11, false);
|
||||
|
||||
assert_eq!(storage.get_client_url_by_machine_id(1, &machine_id), None);
|
||||
assert_eq!(storage.list_clients().len(), 0);
|
||||
assert_eq!(storage.list_all_clients().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_client_authorization_update_does_not_replace_newer_client() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let old_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let new_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(old_token.clone(), 10, true);
|
||||
storage.update_client(new_token.clone(), 20, true);
|
||||
storage.update_client(old_token, 10, false);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(new_token.client_url)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_client_does_not_replace_authorized_route() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(authorized_token.clone(), 10, true);
|
||||
storage.update_client(pending_token, i64::MAX, false);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(authorized_token.client_url)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authorized_client_replaces_pending_route_regardless_of_report_time() {
|
||||
let storage = Storage::new(Db::memory_db().await);
|
||||
let machine_id = uuid::Uuid::new_v4();
|
||||
let pending_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1001");
|
||||
let authorized_token = make_storage_token(1, machine_id, "tcp://127.0.0.1:1002");
|
||||
|
||||
storage.update_client(pending_token, i64::MAX, false);
|
||||
storage.update_client(authorized_token.clone(), 10, true);
|
||||
|
||||
assert_eq!(
|
||||
storage.get_client_url_by_machine_id(1, &machine_id),
|
||||
Some(authorized_token.client_url)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
//! `SeaORM` Entity, hand-written to match the generated entity style.
|
||||
|
||||
use sea_orm::entity::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
|
||||
#[sea_orm(table_name = "managed_config_revisions")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key)]
|
||||
pub id: i32,
|
||||
pub user_id: i32,
|
||||
#[sea_orm(column_type = "Text")]
|
||||
pub device_id: String,
|
||||
#[sea_orm(column_type = "Text")]
|
||||
pub config_revision: String,
|
||||
pub create_time: DateTimeWithTimeZone,
|
||||
pub update_time: DateTimeWithTimeZone,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {
|
||||
#[sea_orm(
|
||||
belongs_to = "super::users::Entity",
|
||||
from = "Column::UserId",
|
||||
to = "super::users::Column::Id",
|
||||
on_update = "Cascade",
|
||||
on_delete = "Cascade"
|
||||
)]
|
||||
Users,
|
||||
}
|
||||
|
||||
impl Related<super::users::Entity> for Entity {
|
||||
fn to() -> RelationDef {
|
||||
Relation::Users.def()
|
||||
}
|
||||
}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
@@ -4,7 +4,6 @@ pub mod prelude;
|
||||
|
||||
pub mod groups;
|
||||
pub mod groups_permissions;
|
||||
pub mod managed_config_revisions;
|
||||
pub mod permissions;
|
||||
pub mod tower_sessions;
|
||||
pub mod user_running_network_configs;
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
pub use super::groups::Entity as Groups;
|
||||
pub use super::groups_permissions::Entity as GroupsPermissions;
|
||||
pub use super::managed_config_revisions::Entity as ManagedConfigRevisions;
|
||||
pub use super::permissions::Entity as Permissions;
|
||||
pub use super::tower_sessions::Entity as TowerSessions;
|
||||
pub use super::user_running_network_configs::Entity as UserRunningNetworkConfigs;
|
||||
|
||||
@@ -141,110 +141,6 @@ impl Db {
|
||||
) -> Result<Option<UserIdInDb>, DbErr> {
|
||||
self.get_user_id(token).await
|
||||
}
|
||||
|
||||
pub async fn get_managed_config_revision(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
) -> Result<Option<String>, DbErr> {
|
||||
use entity::managed_config_revisions as mcr;
|
||||
|
||||
let revision = mcr::Entity::find()
|
||||
.filter(mcr::Column::UserId.eq(user_id))
|
||||
.filter(mcr::Column::DeviceId.eq(device_id.to_string()))
|
||||
.one(self.orm_db())
|
||||
.await?;
|
||||
|
||||
Ok(revision.map(|row| row.config_revision))
|
||||
}
|
||||
|
||||
pub async fn set_managed_config_revision(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
config_revision: &str,
|
||||
) -> Result<(), DbErr> {
|
||||
use entity::managed_config_revisions as mcr;
|
||||
|
||||
let now = chrono::Local::now().fixed_offset();
|
||||
let on_conflict = OnConflict::columns([mcr::Column::UserId, mcr::Column::DeviceId])
|
||||
.update_columns([mcr::Column::ConfigRevision, mcr::Column::UpdateTime])
|
||||
.to_owned();
|
||||
let insert_m = mcr::ActiveModel {
|
||||
user_id: Set(user_id),
|
||||
device_id: Set(device_id.to_string()),
|
||||
config_revision: Set(config_revision.to_string()),
|
||||
create_time: Set(now),
|
||||
update_time: Set(now),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
mcr::Entity::insert(insert_m)
|
||||
.on_conflict(on_conflict)
|
||||
.do_nothing()
|
||||
.exec(self.orm_db())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn insert_or_update_web_network_config(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
network_inst_id: Uuid,
|
||||
network_config: NetworkConfig,
|
||||
) -> Result<bool, DbErr> {
|
||||
let now = chrono::Local::now().fixed_offset();
|
||||
let network_config =
|
||||
serde_json::to_string(&network_config).map_err(|e| DbErr::Json(e.to_string()))?;
|
||||
let source = ConfigSource::Web.as_str();
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO user_running_network_configs (
|
||||
user_id, device_id, network_instance_id, network_config,
|
||||
source, disabled, create_time, update_time
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(user_id, device_id, network_instance_id) DO UPDATE SET
|
||||
network_config = excluded.network_config,
|
||||
source = excluded.source,
|
||||
disabled = excluded.disabled,
|
||||
update_time = excluded.update_time
|
||||
WHERE user_running_network_configs.source = ?
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(device_id.to_string())
|
||||
.bind(network_inst_id.to_string())
|
||||
.bind(network_config)
|
||||
.bind(source)
|
||||
.bind(false)
|
||||
.bind(now)
|
||||
.bind(now)
|
||||
.bind(source)
|
||||
.execute(&self.db)
|
||||
.await
|
||||
.map_err(|e| DbErr::Custom(e.to_string()))?;
|
||||
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
pub async fn delete_web_network_configs(
|
||||
&self,
|
||||
(user_id, device_id): (UserIdInDb, Uuid),
|
||||
network_inst_ids: &[Uuid],
|
||||
) -> Result<(), DbErr> {
|
||||
use entity::user_running_network_configs as urnc;
|
||||
|
||||
urnc::Entity::delete_many()
|
||||
.filter(urnc::Column::UserId.eq(user_id))
|
||||
.filter(urnc::Column::DeviceId.eq(device_id.to_string()))
|
||||
.filter(urnc::Column::Source.eq(ConfigSource::Web.as_str()))
|
||||
.filter(
|
||||
urnc::Column::NetworkInstanceId
|
||||
.is_in(network_inst_ids.iter().map(|id| id.to_string())),
|
||||
)
|
||||
.exec(self.orm_db())
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -572,46 +468,4 @@ mod tests {
|
||||
assert_eq!(device1_configs.len(), 1);
|
||||
assert_eq!(device2_configs.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_web_network_config_does_not_replace_user_owned_config() {
|
||||
let db = Db::memory_db().await;
|
||||
let user_id = db.auto_create_user("user-web-race").await.unwrap().id;
|
||||
let device_id = uuid::Uuid::new_v4();
|
||||
let inst_id = uuid::Uuid::new_v4();
|
||||
|
||||
db.insert_or_update_user_network_config(
|
||||
(user_id, device_id),
|
||||
inst_id,
|
||||
NetworkConfig {
|
||||
network_name: Some("user-owned".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
ConfigSource::User,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let updated = db
|
||||
.insert_or_update_web_network_config(
|
||||
(user_id, device_id),
|
||||
inst_id,
|
||||
NetworkConfig {
|
||||
network_name: Some("web-owned".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!updated);
|
||||
let saved = db
|
||||
.get_network_config((user_id, device_id), &inst_id.to_string())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(saved.get_network_config_source(), ConfigSource::User);
|
||||
let saved_config = saved.get_network_config().unwrap();
|
||||
assert_eq!(saved_config.network_name.as_deref(), Some("user-owned"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
#[macro_use]
|
||||
extern crate rust_i18n;
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::{net::IpAddr, time::Duration};
|
||||
|
||||
use clap::Parser;
|
||||
use easytier::tunnel::websocket::WsTunnelListener;
|
||||
@@ -113,14 +113,6 @@ struct Cli {
|
||||
)]
|
||||
geoip_db: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "ET_HEARTBEAT_MIN_RESPONSE_MS",
|
||||
default_value = "0",
|
||||
help = t!("cli.heartbeat_min_response_ms").to_string(),
|
||||
)]
|
||||
heartbeat_min_response_ms: u64,
|
||||
|
||||
#[cfg(feature = "embed")]
|
||||
#[arg(
|
||||
long,
|
||||
@@ -320,7 +312,6 @@ async fn main() {
|
||||
let mut mgr = client_manager::ClientManager::new(
|
||||
db.clone(),
|
||||
cli.geoip_db,
|
||||
Duration::from_millis(cli.heartbeat_min_response_ms),
|
||||
feature_flags.clone(),
|
||||
webhook_config.clone(),
|
||||
);
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
use sea_orm_migration::prelude::*;
|
||||
|
||||
pub struct Migration;
|
||||
|
||||
impl MigrationName for Migration {
|
||||
fn name(&self) -> &str {
|
||||
"m20260619_000005_managed_config_revisions"
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
manager
|
||||
.get_connection()
|
||||
.execute_unprepared(
|
||||
r#"
|
||||
CREATE TABLE managed_config_revisions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
user_id INTEGER NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
config_revision TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
update_time TEXT NOT NULL,
|
||||
CONSTRAINT fk_managed_config_revisions_user_id_to_users_id
|
||||
FOREIGN KEY (user_id) REFERENCES users(id)
|
||||
ON DELETE CASCADE
|
||||
ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_managed_config_revisions_scope
|
||||
ON managed_config_revisions(user_id, device_id);
|
||||
"#,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
manager
|
||||
.get_connection()
|
||||
.execute_unprepared("DROP TABLE managed_config_revisions;")
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ mod m20241029_000001_init;
|
||||
mod m20260403_000002_scope_network_config_unique;
|
||||
mod m20260421_000003_add_network_config_source;
|
||||
mod m20260514_000004_rename_web_config_source;
|
||||
mod m20260619_000005_managed_config_revisions;
|
||||
|
||||
pub struct Migrator;
|
||||
|
||||
@@ -16,7 +15,6 @@ impl MigratorTrait for Migrator {
|
||||
Box::new(m20260403_000002_scope_network_config_unique::Migration),
|
||||
Box::new(m20260421_000003_add_network_config_source::Migration),
|
||||
Box::new(m20260514_000004_rename_web_config_source::Migration),
|
||||
Box::new(m20260619_000005_managed_config_revisions::Migration),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,7 +307,7 @@ impl RestfulServer {
|
||||
async fn handle_list_all_sessions_internal(
|
||||
State(client_mgr): AppState,
|
||||
) -> Result<Json<ListSessionJsonResp>, HttpHandleError> {
|
||||
let ret = client_mgr.list_all_sessions().await;
|
||||
let ret = client_mgr.list_sessions().await;
|
||||
Ok(ListSessionJsonResp(ret).into())
|
||||
}
|
||||
|
||||
|
||||
@@ -93,8 +93,6 @@ struct ManagedNetworkConfigJson {
|
||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||
struct ReconcileManagedNetworkConfigsJsonReq {
|
||||
managed_network_configs: Vec<ManagedNetworkConfigJson>,
|
||||
config_revision: Option<String>,
|
||||
expected_config_revision: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||
@@ -359,21 +357,13 @@ impl NetworkApi {
|
||||
})
|
||||
.collect();
|
||||
client_mgr
|
||||
.reconcile_managed_network_configs(
|
||||
user_id,
|
||||
machine_id,
|
||||
desired,
|
||||
payload.config_revision,
|
||||
payload.expected_config_revision,
|
||||
)
|
||||
.reconcile_managed_network_configs(user_id, machine_id, desired)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let status = if crate::client_manager::is_managed_config_revision_conflict(&err) {
|
||||
StatusCode::CONFLICT
|
||||
} else {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
};
|
||||
(status, other_error(err.to_string()).into())
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
other_error(err.to_string()).into(),
|
||||
)
|
||||
})?;
|
||||
Ok(Void::default().into())
|
||||
}
|
||||
|
||||
+16
-589
@@ -1,248 +1,6 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::VecDeque,
|
||||
fmt,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
const VALIDATE_TOKEN_INITIAL_CONCURRENCY: usize = 8;
|
||||
const VALIDATE_TOKEN_MIN_CONCURRENCY: usize = 2;
|
||||
const VALIDATE_TOKEN_MAX_CONCURRENCY: usize = 64;
|
||||
const VALIDATE_TOKEN_ADJUST_WINDOW: Duration = Duration::from_secs(1);
|
||||
const VALIDATE_TOKEN_SLOW_THRESHOLD: Duration = Duration::from_secs(2);
|
||||
const WEBHOOK_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
struct AdaptiveValidateLimiter {
|
||||
state: Mutex<AdaptiveValidateLimiterState>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AdaptiveValidateLimiter {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("AdaptiveValidateLimiter")
|
||||
.field("state", &self.lock_state())
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
struct AdaptiveValidateLimiterState {
|
||||
limit: usize,
|
||||
in_flight: usize,
|
||||
waiters: VecDeque<oneshot::Sender<AdaptiveValidateGrant>>,
|
||||
window_started_at: Instant,
|
||||
samples: usize,
|
||||
slow_samples: usize,
|
||||
failures: usize,
|
||||
had_queue: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AdaptiveValidateLimiterState {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("AdaptiveValidateLimiterState")
|
||||
.field("limit", &self.limit)
|
||||
.field("in_flight", &self.in_flight)
|
||||
.field("waiters", &self.waiters.len())
|
||||
.field("window_started_at", &self.window_started_at)
|
||||
.field("samples", &self.samples)
|
||||
.field("slow_samples", &self.slow_samples)
|
||||
.field("failures", &self.failures)
|
||||
.field("had_queue", &self.had_queue)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
struct AdaptiveValidatePermit {
|
||||
limiter: Arc<AdaptiveValidateLimiter>,
|
||||
started_at: Instant,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
struct AdaptiveValidateGrant {
|
||||
limiter: Arc<AdaptiveValidateLimiter>,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LimitAdjustment {
|
||||
Unchanged,
|
||||
Increased,
|
||||
Decreased,
|
||||
}
|
||||
|
||||
impl AdaptiveValidateLimiter {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(AdaptiveValidateLimiterState::new(Instant::now())),
|
||||
})
|
||||
}
|
||||
|
||||
async fn acquire(self: &Arc<Self>) -> AdaptiveValidatePermit {
|
||||
loop {
|
||||
let receiver = {
|
||||
let mut state = self.lock_state();
|
||||
state.complete_window_if_due(Instant::now());
|
||||
if state.waiters.is_empty() && state.in_flight < state.limit {
|
||||
state.in_flight += 1;
|
||||
return AdaptiveValidatePermit::new(self.clone());
|
||||
}
|
||||
|
||||
let (sender, receiver) = oneshot::channel();
|
||||
state.had_queue = true;
|
||||
state.waiters.push_back(sender);
|
||||
self.grant_waiters(&mut state);
|
||||
receiver
|
||||
};
|
||||
|
||||
if let Ok(grant) = receiver.await {
|
||||
return grant.into_permit();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn grant_waiters(self: &Arc<Self>, state: &mut AdaptiveValidateLimiterState) {
|
||||
while state.in_flight < state.limit {
|
||||
let Some(waiter) = state.waiters.pop_front() else {
|
||||
break;
|
||||
};
|
||||
state.in_flight += 1;
|
||||
if let Err(mut grant) = waiter.send(AdaptiveValidateGrant::new(self.clone())) {
|
||||
grant.disarm();
|
||||
state.in_flight -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_sample(self: &Arc<Self>, elapsed: Duration, success: bool) {
|
||||
let mut state = self.lock_state();
|
||||
let adjustment = state.record_sample(Instant::now(), elapsed, success);
|
||||
if adjustment == LimitAdjustment::Increased {
|
||||
self.grant_waiters(&mut state);
|
||||
}
|
||||
}
|
||||
|
||||
fn release_slot(self: &Arc<Self>) {
|
||||
let mut state = self.lock_state();
|
||||
state.in_flight = state.in_flight.saturating_sub(1);
|
||||
self.grant_waiters(&mut state);
|
||||
}
|
||||
|
||||
fn lock_state(&self) -> std::sync::MutexGuard<'_, AdaptiveValidateLimiterState> {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("adaptive validate limiter state should not be poisoned")
|
||||
}
|
||||
}
|
||||
|
||||
impl AdaptiveValidateLimiterState {
|
||||
fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
limit: VALIDATE_TOKEN_INITIAL_CONCURRENCY,
|
||||
in_flight: 0,
|
||||
waiters: VecDeque::new(),
|
||||
window_started_at: now,
|
||||
samples: 0,
|
||||
slow_samples: 0,
|
||||
failures: 0,
|
||||
had_queue: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_sample(&mut self, now: Instant, elapsed: Duration, success: bool) -> LimitAdjustment {
|
||||
self.samples += 1;
|
||||
if elapsed > VALIDATE_TOKEN_SLOW_THRESHOLD {
|
||||
self.slow_samples += 1;
|
||||
}
|
||||
if !success {
|
||||
self.failures += 1;
|
||||
}
|
||||
self.complete_window_if_due(now)
|
||||
}
|
||||
|
||||
fn complete_window_if_due(&mut self, now: Instant) -> LimitAdjustment {
|
||||
if now.duration_since(self.window_started_at) < VALIDATE_TOKEN_ADJUST_WINDOW {
|
||||
return LimitAdjustment::Unchanged;
|
||||
}
|
||||
|
||||
let old_limit = self.limit;
|
||||
if self.samples > 0 {
|
||||
if self.failures > 0 || self.is_p95_slow() {
|
||||
self.limit = (self.limit / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY);
|
||||
} else if self.had_queue {
|
||||
self.limit = (self.limit + 1).min(VALIDATE_TOKEN_MAX_CONCURRENCY);
|
||||
}
|
||||
}
|
||||
|
||||
self.window_started_at = now;
|
||||
self.samples = 0;
|
||||
self.slow_samples = 0;
|
||||
self.failures = 0;
|
||||
self.had_queue = false;
|
||||
|
||||
match self.limit.cmp(&old_limit) {
|
||||
Ordering::Greater => LimitAdjustment::Increased,
|
||||
Ordering::Less => LimitAdjustment::Decreased,
|
||||
Ordering::Equal => LimitAdjustment::Unchanged,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_p95_slow(&self) -> bool {
|
||||
self.slow_samples > 0 && self.slow_samples * 20 >= self.samples
|
||||
}
|
||||
}
|
||||
|
||||
impl AdaptiveValidatePermit {
|
||||
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
|
||||
Self {
|
||||
limiter,
|
||||
started_at: Instant::now(),
|
||||
completed: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn complete(mut self, success: bool) {
|
||||
self.limiter
|
||||
.record_sample(self.started_at.elapsed(), success);
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl AdaptiveValidateGrant {
|
||||
fn new(limiter: Arc<AdaptiveValidateLimiter>) -> Self {
|
||||
Self {
|
||||
limiter,
|
||||
active: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn into_permit(mut self) -> AdaptiveValidatePermit {
|
||||
self.active = false;
|
||||
AdaptiveValidatePermit::new(self.limiter.clone())
|
||||
}
|
||||
|
||||
fn disarm(&mut self) {
|
||||
self.active = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AdaptiveValidateGrant {
|
||||
fn drop(&mut self) {
|
||||
if self.active {
|
||||
self.limiter.release_slot();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AdaptiveValidatePermit {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
self.limiter.record_sample(self.started_at.elapsed(), false);
|
||||
}
|
||||
self.limiter.release_slot();
|
||||
}
|
||||
}
|
||||
|
||||
/// Webhook configuration for external integrations.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -253,7 +11,6 @@ pub struct WebhookConfig {
|
||||
pub web_instance_id: Option<String>,
|
||||
pub web_instance_api_base_url: Option<String>,
|
||||
|
||||
validate_limiter: Arc<AdaptiveValidateLimiter>,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
@@ -271,11 +28,7 @@ impl WebhookConfig {
|
||||
internal_auth_token,
|
||||
web_instance_id,
|
||||
web_instance_api_base_url,
|
||||
validate_limiter: AdaptiveValidateLimiter::new(),
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(WEBHOOK_HTTP_TIMEOUT)
|
||||
.build()
|
||||
.expect("webhook HTTP client should be valid"),
|
||||
client: reqwest::Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -305,8 +58,6 @@ pub struct ValidateTokenRequest {
|
||||
pub web_instance_id: Option<String>,
|
||||
pub web_instance_api_base_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub persisted_config_revision: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub applied_config_revision: Option<String>,
|
||||
}
|
||||
|
||||
@@ -318,6 +69,7 @@ pub struct ValidateTokenResponse {
|
||||
#[serde(default)]
|
||||
pub binding_version: u64,
|
||||
#[serde(default)]
|
||||
pub managed_network_configs: Option<Vec<ManagedNetworkConfig>>,
|
||||
pub config_revision: String,
|
||||
}
|
||||
|
||||
@@ -373,40 +125,21 @@ impl WebhookConfig {
|
||||
pub async fn validate_token(
|
||||
&self,
|
||||
req: &ValidateTokenRequest,
|
||||
) -> anyhow::Result<ValidateTokenResponse> {
|
||||
self.validate_token_with_http_timeout(req, WEBHOOK_HTTP_TIMEOUT)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn validate_token_with_http_timeout(
|
||||
&self,
|
||||
req: &ValidateTokenRequest,
|
||||
http_timeout: Duration,
|
||||
) -> anyhow::Result<ValidateTokenResponse> {
|
||||
let url = self.webhook_endpoint("validate-token")?;
|
||||
let permit = self.validate_limiter.acquire().await;
|
||||
let ret = match tokio::time::timeout(http_timeout, async {
|
||||
let resp = self
|
||||
.client
|
||||
.post(&url)
|
||||
.header("X-Internal-Auth", self.webhook_auth_secret())
|
||||
.json(req)
|
||||
.send()
|
||||
.await?;
|
||||
let resp = self
|
||||
.client
|
||||
.post(&url)
|
||||
.header("X-Internal-Auth", self.webhook_auth_secret())
|
||||
.json(req)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!("webhook validate-token returned status {}", resp.status());
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!("webhook validate-token returned status {}", resp.status());
|
||||
}
|
||||
|
||||
Ok(resp.json().await?)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(ret) => ret,
|
||||
Err(_) => Err(anyhow::anyhow!("webhook validate-token timed out")),
|
||||
};
|
||||
permit.complete(ret.is_ok());
|
||||
ret
|
||||
Ok(resp.json().await?)
|
||||
}
|
||||
|
||||
/// Notify the webhook receiver that a node has connected.
|
||||
@@ -458,319 +191,13 @@ pub type SharedWebhookConfig = Arc<WebhookConfig>;
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{Json, Router, routing::post};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_increases_under_queue_pressure() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
state.had_queue = true;
|
||||
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
state.record_sample(now, Duration::from_millis(50), true);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Increased
|
||||
);
|
||||
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_does_not_increase_without_queue_pressure() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
state.record_sample(now, Duration::from_millis(50), true);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Unchanged
|
||||
);
|
||||
assert_eq!(state.limit, VALIDATE_TOKEN_INITIAL_CONCURRENCY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_reduces_on_failure() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
|
||||
state.record_sample(now, Duration::from_millis(50), false);
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Decreased
|
||||
);
|
||||
assert_eq!(
|
||||
state.limit,
|
||||
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_reduces_on_slow_latency() {
|
||||
let now = Instant::now();
|
||||
let mut state = AdaptiveValidateLimiterState::new(now);
|
||||
|
||||
state.record_sample(
|
||||
now,
|
||||
VALIDATE_TOKEN_SLOW_THRESHOLD + Duration::from_millis(1),
|
||||
true,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
state.complete_window_if_due(now + VALIDATE_TOKEN_ADJUST_WINDOW),
|
||||
LimitAdjustment::Decreased
|
||||
);
|
||||
assert_eq!(
|
||||
state.limit,
|
||||
(VALIDATE_TOKEN_INITIAL_CONCURRENCY / 2).max(VALIDATE_TOKEN_MIN_CONCURRENCY)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_waiter_acquires_after_release() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let waiter_limiter = limiter.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
let permit = waiter_limiter.acquire().await;
|
||||
permit.complete(true);
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!waiter.is_finished());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_releases_when_permit_is_dropped() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let waiter_limiter = limiter.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
let permit = waiter_limiter.acquire().await;
|
||||
permit.complete(true);
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!waiter.is_finished());
|
||||
|
||||
drop(permits.pop().unwrap());
|
||||
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_skips_canceled_waiters() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let waiter_limiter = limiter.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
let permit = waiter_limiter.acquire().await;
|
||||
permit.complete(true);
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!waiter.is_finished());
|
||||
waiter.abort();
|
||||
assert!(waiter.await.unwrap_err().is_cancelled());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
|
||||
let state = limiter.lock_state();
|
||||
assert_eq!(state.samples, 1);
|
||||
assert_eq!(state.failures, 0);
|
||||
drop(state);
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_validate_limiter_releases_dropped_grant_without_failure_sample() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
{
|
||||
let mut state = limiter.lock_state();
|
||||
state.in_flight = 1;
|
||||
}
|
||||
|
||||
drop(AdaptiveValidateGrant::new(limiter.clone()));
|
||||
|
||||
let state = limiter.lock_state();
|
||||
assert_eq!(state.in_flight, 0);
|
||||
assert_eq!(state.samples, 0);
|
||||
assert_eq!(state.failures, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_validate_limiter_wakes_multiple_waiters_in_order() {
|
||||
let limiter = AdaptiveValidateLimiter::new();
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(limiter.acquire().await);
|
||||
}
|
||||
|
||||
let (first_acquired_tx, first_acquired_rx) = oneshot::channel();
|
||||
let (first_release_tx, first_release_rx) = oneshot::channel();
|
||||
let first = {
|
||||
let limiter = limiter.clone();
|
||||
tokio::spawn(async move {
|
||||
let permit = limiter.acquire().await;
|
||||
first_acquired_tx.send(()).unwrap();
|
||||
first_release_rx.await.unwrap();
|
||||
permit.complete(true);
|
||||
})
|
||||
};
|
||||
let (second_acquired_tx, mut second_acquired_rx) = oneshot::channel();
|
||||
let second = {
|
||||
let limiter = limiter.clone();
|
||||
tokio::spawn(async move {
|
||||
let permit = limiter.acquire().await;
|
||||
second_acquired_tx.send(()).unwrap();
|
||||
permit.complete(true);
|
||||
})
|
||||
};
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert!(!first.is_finished());
|
||||
assert!(!second.is_finished());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
tokio::time::timeout(Duration::from_secs(1), first_acquired_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), &mut second_acquired_rx)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
first_release_tx.send(()).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), first)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), &mut second_acquired_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), second)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_token_http_timeout_starts_after_limiter_permit() {
|
||||
let app = Router::new().route(
|
||||
"/validate-token",
|
||||
post(|| async {
|
||||
Json(json!({
|
||||
"valid": true,
|
||||
"config_revision": "rev-1"
|
||||
}))
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let webhook = WebhookConfig::new(Some(format!("http://{addr}")), None, None, None, None);
|
||||
let mut permits = Vec::new();
|
||||
for _ in 0..VALIDATE_TOKEN_INITIAL_CONCURRENCY {
|
||||
permits.push(webhook.validate_limiter.acquire().await);
|
||||
}
|
||||
|
||||
let validate_webhook = webhook.clone();
|
||||
let validate = tokio::spawn(async move {
|
||||
let req = ValidateTokenRequest {
|
||||
token: "token".to_string(),
|
||||
machine_id: uuid::Uuid::new_v4().to_string(),
|
||||
public_ip: None,
|
||||
hostname: String::new(),
|
||||
version: String::new(),
|
||||
os_type: None,
|
||||
os_version: None,
|
||||
os_distribution: None,
|
||||
web_instance_id: None,
|
||||
web_instance_api_base_url: None,
|
||||
persisted_config_revision: None,
|
||||
applied_config_revision: None,
|
||||
};
|
||||
validate_webhook
|
||||
.validate_token_with_http_timeout(&req, Duration::from_millis(20))
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(!validate.is_finished());
|
||||
|
||||
permits.pop().unwrap().complete(true);
|
||||
let resp = tokio::time::timeout(Duration::from_secs(1), validate)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(resp.valid);
|
||||
|
||||
for permit in permits {
|
||||
permit.complete(true);
|
||||
}
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_token_response_deserializes_config_revision() {
|
||||
fn validate_token_response_allows_missing_managed_configs() {
|
||||
let resp: ValidateTokenResponse =
|
||||
serde_json::from_str(r#"{"valid":true,"config_revision":"rev-1"}"#).unwrap();
|
||||
assert!(resp.valid);
|
||||
assert_eq!(resp.config_revision, "rev-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_token_response_allows_missing_config_revision() {
|
||||
let resp: ValidateTokenResponse = serde_json::from_str(r#"{"valid":true}"#).unwrap();
|
||||
assert!(resp.valid);
|
||||
assert!(resp.config_revision.is_empty());
|
||||
assert!(resp.managed_network_configs.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
+13
-11
@@ -28,10 +28,6 @@ path = "src/easytier-cli.rs"
|
||||
name = "easytier"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bench]]
|
||||
name = "tx_throughput"
|
||||
harness = false
|
||||
|
||||
[dependencies]
|
||||
git-version = "0.3.9"
|
||||
|
||||
@@ -56,6 +52,7 @@ toml = "0.8.12"
|
||||
chrono = { version = "0.4.37", features = ["serde"] }
|
||||
|
||||
guarden = "0.2"
|
||||
hotpath = { version = "0.18", default-features = false, optional = true }
|
||||
quanta = "0.12"
|
||||
|
||||
delegate = "0.13.5"
|
||||
@@ -88,6 +85,7 @@ atomic_refcell = "0.1.13"
|
||||
|
||||
quinn = { version = "0.11.8", optional = true, features = ["ring"] }
|
||||
quinn-proto = { version = "0.11.12", optional = true }
|
||||
quinn-udp = { version = "0.5", optional = true }
|
||||
seahash = { version = "4.1.0", optional = true }
|
||||
|
||||
rustls = { version = "0.23.0", features = [
|
||||
@@ -340,7 +338,6 @@ zip = "4.0.0"
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = "0.5.1"
|
||||
serial_test = "3.0.0"
|
||||
rstest = "0.25.0"
|
||||
futures-util = "0.3.31"
|
||||
@@ -365,6 +362,7 @@ default = [
|
||||
"faketcp",
|
||||
"magic-dns",
|
||||
"zstd",
|
||||
"udp-gso",
|
||||
]
|
||||
full = [
|
||||
"websocket",
|
||||
@@ -382,6 +380,7 @@ full = [
|
||||
]
|
||||
wireguard = ["dep:boringtun", "dep:ring"]
|
||||
quic = ["dep:quinn", "dep:quinn-proto", "dep:seahash", "dep:rustls", "dep:rcgen"]
|
||||
udp-gso = ["dep:quinn-udp"]
|
||||
kcp = ["dep:kcp-sys"]
|
||||
mimalloc = ["dep:mimalloc"]
|
||||
aes-gcm = ["dep:aes-gcm"]
|
||||
@@ -407,14 +406,17 @@ jemalloc-prof = [
|
||||
"jemalloc-sys/stats",
|
||||
]
|
||||
tracing = ["tokio/tracing", "dep:console-subscriber"]
|
||||
hotpath = [
|
||||
"dep:hotpath",
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"hotpath/parking_lot",
|
||||
"hotpath/flume",
|
||||
]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
magic-dns = ["dep:hickory-client", "dep:hickory-server"]
|
||||
faketcp = ["dep:flume"]
|
||||
zstd = ["dep:zstd"]
|
||||
# Deprecated: hotpath profiling has been removed. These feature aliases are
|
||||
# retained as no-ops so existing build scripts using `--features hotpath*`
|
||||
# continue to work without pulling in any dependencies.
|
||||
hotpath = []
|
||||
hotpath-cpu = ["hotpath"]
|
||||
hotpath-alloc = ["hotpath"]
|
||||
# For Network Extension on macOS
|
||||
macos-ne = []
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
# TX Throughput Benchmark
|
||||
|
||||
Criterion benchmark for EasyTier's TX injection path (`peer_manager::send_msg_by_ip`).
|
||||
|
||||
## What it measures
|
||||
|
||||
The benchmark sets up two EasyTier instances (`hot-a` / `hot-b`) and drives
|
||||
packets from `hot-a` to `hot-b` via `peer_manager.send_msg_by_ip`. This is the
|
||||
same entry point `easytier-core` uses for daily forwarded traffic, so the
|
||||
numbers reflect the real TX hot path: NIC pipeline → route lookup →
|
||||
compress/encrypt → peer connection → tunnel send.
|
||||
|
||||
Two variants are reported per tunnel kind:
|
||||
|
||||
| Bench | What it measures |
|
||||
| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `tx_throughput/<tunnel>` | Serial baseline. One send in flight at a time. Reports per-packet CPU cost (TX injection latency). |
|
||||
| `tx_throughput/<tunnel>-saturate` | Spawns `TX_THROUGHPUT_INFLIGHT` tokio tasks that independently pump `send_msg_by_ip`. Reports the aggregate throughput ceiling the peer manager + tunnel can sustain across worker threads. |
|
||||
|
||||
> **Out of scope (by design):** TUN read/write (`no_tun = true`), compression
|
||||
> (default `None`), reverse/RX-side measurement, multi-peer fanout. Add
|
||||
> separate benchmarks if you need those.
|
||||
|
||||
## Quick start
|
||||
|
||||
### ring tunnel (no root, fastest)
|
||||
|
||||
```bash
|
||||
cargo bench --bench tx_throughput
|
||||
```
|
||||
|
||||
Smoke run (faster iteration):
|
||||
|
||||
```bash
|
||||
TX_THROUGHPUT_MEASUREMENT_SECS=2 \
|
||||
TX_THROUGHPUT_WARMUP_SECS=1 \
|
||||
TX_THROUGHPUT_SAMPLE_SIZE=10 \
|
||||
cargo bench --bench tx_throughput -- --quiet
|
||||
```
|
||||
|
||||
### tcp / udp tunnels (requires Docker + root)
|
||||
|
||||
The benchmark creates a Docker network and registers each container's netns
|
||||
under `/var/run/netns`, which requires root. Run the whole command under
|
||||
`sudo`:
|
||||
|
||||
```bash
|
||||
sudo TX_THROUGHPUT_TUNNEL=tcp \
|
||||
TX_THROUGHPUT_MEASUREMENT_SECS=5 \
|
||||
TX_THROUGHPUT_WARMUP_SECS=2 \
|
||||
TX_THROUGHPUT_INFLIGHT=64 \
|
||||
cargo bench --bench tx_throughput -- --quiet
|
||||
|
||||
sudo TX_THROUGHPUT_TUNNEL=udp cargo bench --bench tx_throughput -- --quiet
|
||||
```
|
||||
|
||||
> If `sudo` cannot find `cargo`, use `sudo -E` or the absolute path
|
||||
> (`$(which cargo)`).
|
||||
|
||||
## Environment variables
|
||||
|
||||
| Variable | Default | Notes |
|
||||
| -------------------------------- | --------------------- | -------------------------------------- |
|
||||
| `TX_THROUGHPUT_TUNNEL` | `ring` | `ring` / `tcp` / `udp` |
|
||||
| `TX_THROUGHPUT_PKT_SIZE` | `1400` | IP total length in bytes |
|
||||
| `TX_THROUGHPUT_WORKER_THREADS` | `4` | tokio worker threads |
|
||||
| `TX_THROUGHPUT_INFLIGHT` | `64` | saturate-mode concurrency (task count) |
|
||||
| `TX_THROUGHPUT_TUNNEL_PORT` | `35521` | tcp/udp listen port |
|
||||
| `TX_THROUGHPUT_MEASUREMENT_SECS` | `10` | Criterion `measurement_time` |
|
||||
| `TX_THROUGHPUT_WARMUP_SECS` | `3` | Criterion `warm_up_time` |
|
||||
| `TX_THROUGHPUT_SAMPLE_SIZE` | `10` | Criterion `sample_size` (min 10) |
|
||||
| `TX_THROUGHPUT_DOCKER_IMAGE` | `busybox:latest` | tcp/udp only |
|
||||
| `TX_THROUGHPUT_DOCKER_NET` | `easytier-bench-<id>` | auto-generated unique name |
|
||||
| `TX_THROUGHPUT_DOCKER_SUBNET` | `172.31.250.0/24` | |
|
||||
| `TX_THROUGHPUT_DOCKER_IP_A` | `172.31.250.2` | |
|
||||
| `TX_THROUGHPUT_DOCKER_IP_B` | `172.31.250.3` | |
|
||||
|
||||
## Parameter sweeps
|
||||
|
||||
```bash
|
||||
# Packet size
|
||||
for sz in 64 256 1400 9000; do
|
||||
TX_THROUGHPUT_PKT_SIZE=$sz cargo bench --bench tx_throughput -- --quick
|
||||
done
|
||||
|
||||
# Inflight depth (self-check: depth=1 should match serial baseline)
|
||||
for d in 1 4 16 64 256; do
|
||||
TX_THROUGHPUT_INFLIGHT=$d cargo bench --bench tx_throughput -- --quick
|
||||
done
|
||||
|
||||
# Worker threads
|
||||
for w in 1 2 4 8; do
|
||||
TX_THROUGHPUT_WORKER_THREADS=$w cargo bench --bench tx_throughput -- --quick
|
||||
done
|
||||
```
|
||||
|
||||
## Interpreting results
|
||||
|
||||
- **`<tunnel>`** reports per-packet latency. Lower is better. Throughput
|
||||
column here is "what one in-flight sender sustains".
|
||||
- **`<tunnel>-saturate`** reports aggregate throughput across
|
||||
`TX_THROUGHPUT_INFLIGHT` concurrent senders. If this matches the serial
|
||||
baseline, the TX path is bottlenecked on an internal serialization point
|
||||
(lock, single-threaded queue, etc.) rather than CPU or link bandwidth.
|
||||
|
||||
### Known finding (ring, single peer)
|
||||
|
||||
On the ring tunnel with a single destination peer, saturate does **not** beat
|
||||
serial (observed ~277 MiB/s saturate vs ~288 MiB/s serial on a 4-worker
|
||||
runtime). This points to a serialization point inside the peer-connection TX
|
||||
path. Tunnels with real I/O await points (tcp/udp via Docker) are expected to
|
||||
show a saturate > serial gap; verify with the sudo commands above.
|
||||
|
||||
## Output artifacts
|
||||
|
||||
Criterion writes HTML reports + SVG plots under
|
||||
`easytier/target/criterion/`. Open `tx_throughput/<tunnel>/report/index.html`
|
||||
or `.../<tunnel>-saturate/report/index.html` in a browser to inspect
|
||||
distributions and regressions across runs.
|
||||
@@ -1,472 +0,0 @@
|
||||
use std::{
|
||||
net::IpAddr,
|
||||
path::PathBuf,
|
||||
process::{Command, Stdio},
|
||||
str::FromStr,
|
||||
sync::Arc,
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use bytes::BytesMut;
|
||||
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
|
||||
|
||||
use easytier::{
|
||||
common::config::{ConfigLoader, TomlConfigLoader},
|
||||
instance::instance::Instance,
|
||||
tunnel::{
|
||||
packet_def::ZCPacket, ring::RingTunnelConnector, tcp::TcpTunnelConnector,
|
||||
udp::UdpTunnelConnector,
|
||||
},
|
||||
};
|
||||
|
||||
const VIRTUAL_IP_A: &str = "10.144.144.1";
|
||||
const VIRTUAL_IP_B: &str = "10.144.144.2";
|
||||
const DEFAULT_DOCKER_SUBNET: &str = "172.31.250.0/24";
|
||||
const DEFAULT_DOCKER_IP_A: &str = "172.31.250.2";
|
||||
const DEFAULT_DOCKER_IP_B: &str = "172.31.250.3";
|
||||
const DEFAULT_TUNNEL_PORT: u16 = 35521;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum TunnelKind {
|
||||
Ring,
|
||||
Tcp,
|
||||
Udp,
|
||||
}
|
||||
|
||||
impl TunnelKind {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
TunnelKind::Ring => "ring",
|
||||
TunnelKind::Tcp => "tcp",
|
||||
TunnelKind::Udp => "udp",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for TunnelKind {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
match value {
|
||||
"ring" => Ok(TunnelKind::Ring),
|
||||
"tcp" => Ok(TunnelKind::Tcp),
|
||||
"udp" => Ok(TunnelKind::Udp),
|
||||
other => Err(format!(
|
||||
"unsupported TX_THROUGHPUT_TUNNEL={other:?}; expected ring, tcp, or udp"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BenchTopology {
|
||||
_docker: Option<DockerNetns>,
|
||||
inst_a: Instance,
|
||||
_inst_b: Instance,
|
||||
dst: IpAddr,
|
||||
packet: ZCPacket,
|
||||
}
|
||||
|
||||
struct DockerNetns {
|
||||
network: String,
|
||||
container_a: String,
|
||||
container_b: String,
|
||||
netns_a: String,
|
||||
netns_b: String,
|
||||
ip_a: String,
|
||||
netns_a_path: PathBuf,
|
||||
netns_b_path: PathBuf,
|
||||
}
|
||||
|
||||
impl DockerNetns {
|
||||
fn create() -> Self {
|
||||
let id = unique_id();
|
||||
let image = env_string("TX_THROUGHPUT_DOCKER_IMAGE", "busybox:latest");
|
||||
let network = env_string("TX_THROUGHPUT_DOCKER_NET", &format!("easytier-bench-{id}"));
|
||||
let subnet = env_string("TX_THROUGHPUT_DOCKER_SUBNET", DEFAULT_DOCKER_SUBNET);
|
||||
let ip_a = env_string("TX_THROUGHPUT_DOCKER_IP_A", DEFAULT_DOCKER_IP_A);
|
||||
let ip_b = env_string("TX_THROUGHPUT_DOCKER_IP_B", DEFAULT_DOCKER_IP_B);
|
||||
let container_a = format!("easytier-bench-a-{id}");
|
||||
let container_b = format!("easytier-bench-b-{id}");
|
||||
let netns_a = format!("easytier-bench-a-{id}");
|
||||
let netns_b = format!("easytier-bench-b-{id}");
|
||||
|
||||
docker(&[
|
||||
"network", "create", "--driver", "bridge", "--subnet", &subnet, &network,
|
||||
]);
|
||||
|
||||
let mut docker_netns = Self {
|
||||
network,
|
||||
container_a,
|
||||
container_b,
|
||||
netns_a,
|
||||
netns_b,
|
||||
ip_a: ip_a.clone(),
|
||||
netns_a_path: PathBuf::new(),
|
||||
netns_b_path: PathBuf::new(),
|
||||
};
|
||||
|
||||
docker_netns.start_container(&docker_netns.container_a, &ip_a, &image);
|
||||
docker_netns.start_container(&docker_netns.container_b, &ip_b, &image);
|
||||
|
||||
let pid_a = docker(&["inspect", "-f", "{{.State.Pid}}", &docker_netns.container_a]);
|
||||
let pid_b = docker(&["inspect", "-f", "{{.State.Pid}}", &docker_netns.container_b]);
|
||||
|
||||
docker_netns.netns_a_path = register_netns(&docker_netns.netns_a, &pid_a);
|
||||
docker_netns.netns_b_path = register_netns(&docker_netns.netns_b, &pid_b);
|
||||
docker_netns
|
||||
}
|
||||
|
||||
fn start_container(&self, name: &str, ip: &str, image: &str) {
|
||||
docker(&[
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
name,
|
||||
"--network",
|
||||
&self.network,
|
||||
"--ip",
|
||||
ip,
|
||||
image,
|
||||
"sleep",
|
||||
"3600",
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DockerNetns {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.netns_a_path);
|
||||
let _ = std::fs::remove_file(&self.netns_b_path);
|
||||
docker_ignore(&["rm", "-f", &self.container_a, &self.container_b]);
|
||||
docker_ignore(&["network", "rm", &self.network]);
|
||||
}
|
||||
}
|
||||
|
||||
fn bench_tx_throughput(c: &mut Criterion) {
|
||||
let tunnel = env_string("TX_THROUGHPUT_TUNNEL", "ring")
|
||||
.parse::<TunnelKind>()
|
||||
.unwrap_or_else(|err| panic!("{err}"));
|
||||
let packet_size = env_parse("TX_THROUGHPUT_PKT_SIZE", 1400usize);
|
||||
const MIN_PKT_SIZE: usize = 28; // IPv4 (20) + UDP (8) header
|
||||
assert!(
|
||||
packet_size >= MIN_PKT_SIZE,
|
||||
"TX_THROUGHPUT_PKT_SIZE={packet_size} is smaller than the minimum {MIN_PKT_SIZE} (IPv4+UDP headers)"
|
||||
);
|
||||
let worker_threads = env_parse("TX_THROUGHPUT_WORKER_THREADS", 4usize);
|
||||
let inflight_depth = env_parse("TX_THROUGHPUT_INFLIGHT", 64usize).max(1);
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(worker_threads)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("create tokio runtime");
|
||||
|
||||
let topology = runtime.block_on(setup_topology(tunnel, packet_size));
|
||||
let peer_manager = topology.inst_a.get_peer_manager();
|
||||
let packet = topology.packet.clone();
|
||||
let dst = topology.dst;
|
||||
|
||||
eprintln!(
|
||||
"tx_throughput: tunnel={} inflight={} workers={} pkt_size={}",
|
||||
tunnel.as_str(),
|
||||
inflight_depth.max(1),
|
||||
worker_threads,
|
||||
packet_size
|
||||
);
|
||||
|
||||
let mut group = c.benchmark_group("tx_throughput");
|
||||
group.throughput(Throughput::Bytes(packet_size as u64));
|
||||
|
||||
// Serial baseline: one packet in flight at a time.
|
||||
// Measures per-packet CPU cost (TX injection latency).
|
||||
group.bench_function(tunnel.as_str(), |b| {
|
||||
b.iter_custom(|iterations| {
|
||||
let pm = peer_manager.clone();
|
||||
let pkt = packet.clone();
|
||||
runtime.block_on(async move {
|
||||
let start = Instant::now();
|
||||
for _ in 0..iterations {
|
||||
pm.send_msg_by_ip(pkt.clone(), dst, false)
|
||||
.await
|
||||
.expect("send packet by EasyTier IP");
|
||||
}
|
||||
start.elapsed()
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
// Saturate: spawn TX_THROUGHPUT_INFLIGHT worker tasks, each independently
|
||||
// pumping send_msg_by_ip. Work is distributed across tokio worker threads,
|
||||
// exposing the peer manager + tunnel's true aggregate throughput ceiling.
|
||||
// With TX_THROUGHPUT_INFLIGHT=1 it degrades to the serial baseline.
|
||||
group.bench_function(format!("{}-saturate", tunnel.as_str()), |b| {
|
||||
b.iter_custom(|iterations| {
|
||||
let pm = peer_manager.clone();
|
||||
let pkt = packet.clone();
|
||||
let concurrency = inflight_depth.min(iterations as usize).max(1);
|
||||
runtime.block_on(async move {
|
||||
let counter = Arc::new(AtomicU64::new(iterations));
|
||||
let start = Instant::now();
|
||||
let mut handles = Vec::with_capacity(concurrency);
|
||||
for _ in 0..concurrency {
|
||||
let pm = pm.clone();
|
||||
let pkt = pkt.clone();
|
||||
let counter = counter.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
loop {
|
||||
if counter
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |cur| {
|
||||
if cur > 0 { Some(cur - 1) } else { None }
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
pm.send_msg_by_ip(pkt.clone(), dst, false)
|
||||
.await
|
||||
.expect("send packet by EasyTier IP");
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.await.expect("saturate worker task panicked");
|
||||
}
|
||||
start.elapsed()
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
group.finish();
|
||||
|
||||
runtime.block_on(async move {
|
||||
drop(topology);
|
||||
});
|
||||
}
|
||||
|
||||
async fn setup_topology(tunnel: TunnelKind, packet_size: usize) -> BenchTopology {
|
||||
let tunnel_port = env_parse("TX_THROUGHPUT_TUNNEL_PORT", DEFAULT_TUNNEL_PORT);
|
||||
let docker = match tunnel {
|
||||
TunnelKind::Ring => None,
|
||||
TunnelKind::Tcp | TunnelKind::Udp => Some(DockerNetns::create()),
|
||||
};
|
||||
|
||||
let (netns_a, netns_b) = match &docker {
|
||||
Some(docker) => (Some(docker.netns_a.clone()), Some(docker.netns_b.clone())),
|
||||
None => (None, None),
|
||||
};
|
||||
let listeners_a = match tunnel {
|
||||
TunnelKind::Ring => Vec::new(),
|
||||
TunnelKind::Tcp | TunnelKind::Udp => vec![
|
||||
format!("{}://0.0.0.0:{}", tunnel.as_str(), tunnel_port)
|
||||
.parse()
|
||||
.unwrap(),
|
||||
],
|
||||
};
|
||||
|
||||
let mut inst_a = Instance::new(no_tun_config("hot-a", VIRTUAL_IP_A, netns_a, listeners_a));
|
||||
let mut inst_b = Instance::new(no_tun_config("hot-b", VIRTUAL_IP_B, netns_b, Vec::new()));
|
||||
|
||||
inst_a.run().await.expect("inst_a run");
|
||||
inst_b.run().await.expect("inst_b run");
|
||||
|
||||
match tunnel {
|
||||
TunnelKind::Ring => inst_b
|
||||
.get_conn_manager()
|
||||
.add_connector(RingTunnelConnector::new(
|
||||
format!("ring://{}", inst_a.id()).parse().unwrap(),
|
||||
)),
|
||||
TunnelKind::Tcp => inst_b
|
||||
.get_conn_manager()
|
||||
.add_connector(TcpTunnelConnector::new(
|
||||
format!(
|
||||
"tcp://{}:{}",
|
||||
docker.as_ref().expect("tcp benchmark needs Docker").ip_a,
|
||||
tunnel_port
|
||||
)
|
||||
.parse()
|
||||
.unwrap(),
|
||||
)),
|
||||
TunnelKind::Udp => inst_b
|
||||
.get_conn_manager()
|
||||
.add_connector(UdpTunnelConnector::new(
|
||||
format!(
|
||||
"udp://{}:{}",
|
||||
docker.as_ref().expect("udp benchmark needs Docker").ip_a,
|
||||
tunnel_port
|
||||
)
|
||||
.parse()
|
||||
.unwrap(),
|
||||
)),
|
||||
}
|
||||
|
||||
wait_for_routes(&inst_a, &inst_b).await;
|
||||
|
||||
BenchTopology {
|
||||
_docker: docker,
|
||||
inst_a,
|
||||
_inst_b: inst_b,
|
||||
dst: VIRTUAL_IP_B.parse().unwrap(),
|
||||
packet: make_data_packet(VIRTUAL_IP_A, VIRTUAL_IP_B, packet_size),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_routes(inst_a: &Instance, inst_b: &Instance) {
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
loop {
|
||||
let routes_a = inst_a.get_peer_manager().list_routes().await;
|
||||
let routes_b = inst_b.get_peer_manager().list_routes().await;
|
||||
if !routes_a.is_empty() && !routes_b.is_empty() {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("EasyTier routes did not converge within 15s");
|
||||
}
|
||||
|
||||
fn make_data_packet(src: &str, dst: &str, total_size: usize) -> ZCPacket {
|
||||
use std::net::Ipv4Addr;
|
||||
|
||||
let hdr_len = 28;
|
||||
let payload_len = total_size.saturating_sub(hdr_len);
|
||||
let ip_total_len = (hdr_len + payload_len) as u16;
|
||||
let mut buf = BytesMut::with_capacity(total_size);
|
||||
|
||||
buf.extend_from_slice(&[
|
||||
0x45,
|
||||
0x00,
|
||||
(ip_total_len >> 8) as u8,
|
||||
(ip_total_len & 0xff) as u8,
|
||||
0x00,
|
||||
0x00,
|
||||
0x40,
|
||||
0x00,
|
||||
0x40,
|
||||
0x11,
|
||||
0x00,
|
||||
0x00,
|
||||
]);
|
||||
let src: Ipv4Addr = src.parse().unwrap();
|
||||
buf.extend_from_slice(&src.octets());
|
||||
let dst: Ipv4Addr = dst.parse().unwrap();
|
||||
buf.extend_from_slice(&dst.octets());
|
||||
|
||||
let udp_len = (8 + payload_len) as u16;
|
||||
buf.extend_from_slice(&[
|
||||
0x30,
|
||||
0x39,
|
||||
0xd4,
|
||||
0x31,
|
||||
(udp_len >> 8) as u8,
|
||||
(udp_len & 0xff) as u8,
|
||||
0x00,
|
||||
0x00,
|
||||
]);
|
||||
|
||||
buf.resize(total_size, 0xaa);
|
||||
ZCPacket::new_with_payload(&buf)
|
||||
}
|
||||
|
||||
fn no_tun_config(
|
||||
name: &str,
|
||||
ipv4: &str,
|
||||
netns: Option<String>,
|
||||
listeners: Vec<url::Url>,
|
||||
) -> TomlConfigLoader {
|
||||
let config = TomlConfigLoader::default();
|
||||
config.set_inst_name(name.to_owned());
|
||||
config.set_netns(netns);
|
||||
config.set_ipv4(Some(ipv4.parse().unwrap()));
|
||||
config.set_listeners(listeners);
|
||||
let mut flags = config.get_flags();
|
||||
flags.no_tun = true;
|
||||
config.set_flags(flags);
|
||||
config
|
||||
}
|
||||
|
||||
fn register_netns(name: &str, pid: &str) -> PathBuf {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let dir = PathBuf::from("/var/run/netns");
|
||||
std::fs::create_dir_all(&dir).expect("create /var/run/netns");
|
||||
let path = dir.join(name);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
std::os::unix::fs::symlink(format!("/proc/{pid}/ns/net"), &path)
|
||||
.expect("link Docker netns into /var/run/netns");
|
||||
path
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = (name, pid);
|
||||
panic!("Docker netns benchmark requires Linux");
|
||||
}
|
||||
}
|
||||
|
||||
fn docker(args: &[&str]) -> String {
|
||||
let output = Command::new("docker")
|
||||
.args(args)
|
||||
.output()
|
||||
.unwrap_or_else(|err| panic!("failed to run docker {args:?}: {err}"));
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"docker {:?} failed with status {:?}: {}",
|
||||
args,
|
||||
output.status.code(),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_owned()
|
||||
}
|
||||
|
||||
fn docker_ignore(args: &[&str]) {
|
||||
let _ = Command::new("docker")
|
||||
.args(args)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status();
|
||||
}
|
||||
|
||||
fn env_string(name: &str, default: &str) -> String {
|
||||
std::env::var(name).unwrap_or_else(|_| default.to_owned())
|
||||
}
|
||||
|
||||
fn env_parse<T>(name: &str, default: T) -> T
|
||||
where
|
||||
T: FromStr,
|
||||
T::Err: std::fmt::Display,
|
||||
{
|
||||
match std::env::var(name) {
|
||||
Ok(value) => value
|
||||
.parse()
|
||||
.unwrap_or_else(|err| panic!("invalid {name}={value:?}: {err}")),
|
||||
Err(_) => default,
|
||||
}
|
||||
}
|
||||
|
||||
fn criterion_config() -> Criterion {
|
||||
let measurement_secs = env_parse("TX_THROUGHPUT_MEASUREMENT_SECS", 10u64);
|
||||
let warmup_secs = env_parse("TX_THROUGHPUT_WARMUP_SECS", 3u64);
|
||||
let sample_size = env_parse("TX_THROUGHPUT_SAMPLE_SIZE", 10usize).max(10);
|
||||
|
||||
Criterion::default()
|
||||
.measurement_time(Duration::from_secs(measurement_secs))
|
||||
.warm_up_time(Duration::from_secs(warmup_secs))
|
||||
.sample_size(sample_size)
|
||||
}
|
||||
|
||||
fn unique_id() -> String {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock before UNIX epoch")
|
||||
.as_nanos();
|
||||
format!("{}-{nanos}", std::process::id())
|
||||
}
|
||||
|
||||
criterion_group! {
|
||||
name = benches;
|
||||
config = criterion_config();
|
||||
targets = bench_tx_throughput
|
||||
}
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,232 @@
|
||||
//! CPU hotspot benchmark for hotpath profiling.
|
||||
//!
|
||||
//! Builds two no-tun EasyTier instances connected via an in-process ring
|
||||
//! tunnel, lets routes converge, then floods data-plane packets through
|
||||
//! `send_msg_by_ip` so that `hotpath-cpu` / samply can collect meaningful
|
||||
//! CPU samples.
|
||||
//!
|
||||
//! Build & run:
|
||||
//! cargo run --profile hotpath --features hotpath,hotpath-cpu \
|
||||
//! --example cpu_hotspot_ring
|
||||
//!
|
||||
//! Prerequisites: hotpath-samply + samply must be installed and on PATH.
|
||||
//! See bench/006-hotpath-cpu-top.md for install instructions.
|
||||
//!
|
||||
//! Then in another terminal:
|
||||
//! hotpath console
|
||||
|
||||
#[cfg(feature = "mimalloc")]
|
||||
#[global_allocator]
|
||||
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
#[cfg(feature = "jemalloc")]
|
||||
#[global_allocator]
|
||||
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::BytesMut;
|
||||
|
||||
use easytier::common::config::{ConfigLoader, PeerConfig, TomlConfigLoader};
|
||||
use easytier::instance::instance::Instance;
|
||||
use easytier::tunnel::packet_def::ZCPacket;
|
||||
use easytier::tunnel::ring::RingTunnelConnector;
|
||||
use easytier::tunnel::udp::UdpTunnelConnector;
|
||||
|
||||
#[tokio::main(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[cfg_attr(feature = "hotpath", hotpath::main)]
|
||||
async fn main() {
|
||||
let duration = std::env::var("HOTPATH_BENCH_SECS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(30u64);
|
||||
|
||||
let pkt_size: usize = std::env::var("HOTPATH_PKT_SIZE")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1400);
|
||||
|
||||
let tunnel_type = std::env::var("HOTPATH_TUNNEL")
|
||||
.ok()
|
||||
.unwrap_or_else(|| "ring".to_string());
|
||||
|
||||
let (inst_a_config, inst_b_config) = match tunnel_type.as_str() {
|
||||
"udp" => {
|
||||
let mut a = no_tun_config("hot-a", "10.144.144.1");
|
||||
a.set_listeners(vec!["udp://0.0.0.0:35521".parse().unwrap()]);
|
||||
(a, no_tun_config("hot-b", "10.144.144.2"))
|
||||
}
|
||||
"tcp" => {
|
||||
let mut a = no_tun_config("hot-a", "10.144.144.1");
|
||||
a.set_listeners(vec!["tcp://0.0.0.0:35522".parse().unwrap()]);
|
||||
(a, no_tun_config("hot-b", "10.144.144.2"))
|
||||
}
|
||||
_ => (
|
||||
no_tun_config("hot-a", "10.144.144.1"),
|
||||
no_tun_config("hot-b", "10.144.144.2"),
|
||||
),
|
||||
};
|
||||
|
||||
let mut inst_a = Instance::new(inst_a_config);
|
||||
let mut inst_b = Instance::new(inst_b_config);
|
||||
|
||||
inst_a.run().await.expect("inst_a run");
|
||||
inst_b.run().await.expect("inst_b run");
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
|
||||
match tunnel_type.as_str() {
|
||||
"ring" => {
|
||||
let ring_url = format!("ring://{}", inst_a.id());
|
||||
inst_b
|
||||
.get_conn_manager()
|
||||
.add_connector(RingTunnelConnector::new(ring_url.parse().unwrap()));
|
||||
}
|
||||
"udp" => {
|
||||
inst_b.get_conn_manager().add_connector(
|
||||
UdpTunnelConnector::new("udp://127.0.0.1:35521".parse().unwrap()),
|
||||
);
|
||||
}
|
||||
"tcp" => {
|
||||
inst_b.get_conn_manager().add_connector(
|
||||
easytier::tunnel::tcp::TcpTunnelConnector::new(
|
||||
"tcp://127.0.0.1:35522".parse().unwrap(),
|
||||
),
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let dst: IpAddr = "10.144.144.2".parse().unwrap();
|
||||
let src = "10.144.144.1";
|
||||
|
||||
let converged = tokio::time::timeout(Duration::from_secs(15), async {
|
||||
loop {
|
||||
let a = inst_a.get_peer_manager().list_routes().await;
|
||||
let b = inst_b.get_peer_manager().list_routes().await;
|
||||
if a.len() >= 1 && b.len() >= 1 {
|
||||
return true;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok();
|
||||
|
||||
if !converged {
|
||||
eprintln!("warning: routes did not converge within 15s");
|
||||
}
|
||||
|
||||
println!(
|
||||
"cpu_hotspot_ring: flooding {}s, pkt_size={}, tunnel={} (converged={})",
|
||||
duration, pkt_size, tunnel_type, converged
|
||||
);
|
||||
|
||||
let pm = inst_a.get_peer_manager();
|
||||
let send_pkt = make_data_packet(src, "10.144.144.2", pkt_size);
|
||||
|
||||
let batch_threshold: u32 = std::env::var("HOTPATH_BATCH")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
// After convergence, enable batch flush for writev optimization
|
||||
if converged && batch_threshold > 1 {
|
||||
pm.set_peer_conn_batch_threshold(batch_threshold);
|
||||
println!("cpu_hotspot_ring: batch_threshold={}", batch_threshold);
|
||||
}
|
||||
|
||||
let pipeline_depth: usize = std::env::var("HOTPATH_PIPELINE")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
println!(
|
||||
"cpu_hotspot_ring: pipeline_depth={}",
|
||||
pipeline_depth
|
||||
);
|
||||
|
||||
let sender_task = tokio::spawn(async move {
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
|
||||
let mut sent: u64 = 0;
|
||||
let start = Instant::now();
|
||||
let mut in_flight = FuturesUnordered::new();
|
||||
|
||||
loop {
|
||||
while in_flight.len() < pipeline_depth {
|
||||
let pkt = send_pkt.clone();
|
||||
in_flight.push(pm.send_msg_by_ip(pkt, dst, false));
|
||||
}
|
||||
in_flight.next().await;
|
||||
sent += 1;
|
||||
if sent % 10000 == 0 {
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
let pps = sent as f64 / elapsed;
|
||||
let mbps = pps * pkt_size as f64 * 8.0 / 1_000_000.0;
|
||||
println!("sent {} pkts ({:.0} pps, {:.0} Mbps)", sent, pps, mbps);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(duration)).await;
|
||||
sender_task.abort();
|
||||
|
||||
println!("cpu_hotspot_ring: done");
|
||||
}
|
||||
|
||||
fn make_data_packet(src: &str, dst: &str, total_size: usize) -> ZCPacket {
|
||||
use std::net::Ipv4Addr;
|
||||
|
||||
let hdr_len = 28;
|
||||
let payload_len = total_size.saturating_sub(hdr_len);
|
||||
let ip_total_len = (hdr_len + payload_len) as u16;
|
||||
|
||||
let mut buf = BytesMut::with_capacity(total_size);
|
||||
|
||||
buf.extend_from_slice(&[
|
||||
0x45,
|
||||
0x00,
|
||||
(ip_total_len >> 8) as u8,
|
||||
(ip_total_len & 0xff) as u8,
|
||||
0x00,
|
||||
0x00,
|
||||
0x40,
|
||||
0x00,
|
||||
0x40,
|
||||
0x11,
|
||||
0x00,
|
||||
0x00,
|
||||
]);
|
||||
let src: Ipv4Addr = src.parse().unwrap();
|
||||
buf.extend_from_slice(&src.octets());
|
||||
let dst: Ipv4Addr = dst.parse().unwrap();
|
||||
buf.extend_from_slice(&dst.octets());
|
||||
|
||||
let udp_len = (8 + payload_len) as u16;
|
||||
buf.extend_from_slice(&[
|
||||
0x30,
|
||||
0x39,
|
||||
0xD4,
|
||||
0x31,
|
||||
(udp_len >> 8) as u8,
|
||||
(udp_len & 0xff) as u8,
|
||||
0x00,
|
||||
0x00,
|
||||
]);
|
||||
|
||||
buf.resize(total_size, 0xAA);
|
||||
|
||||
ZCPacket::new_with_payload(&buf)
|
||||
}
|
||||
|
||||
fn no_tun_config(name: &str, ipv4: &str) -> TomlConfigLoader {
|
||||
let config = TomlConfigLoader::default();
|
||||
config.set_inst_name(name.to_owned());
|
||||
config.set_ipv4(Some(ipv4.parse().unwrap()));
|
||||
let mut flags = config.get_flags();
|
||||
flags.no_tun = true;
|
||||
config.set_flags(flags);
|
||||
config
|
||||
}
|
||||
@@ -6,7 +6,7 @@ use std::{
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
|
||||
use crate::common::{config::ConfigLoader, global_ctx::ArcGlobalCtx, token_bucket::TokenBucket};
|
||||
use crate::proto::acl::*;
|
||||
|
||||
@@ -129,6 +129,7 @@ impl Compressor for DefaultCompressor {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "DefaultCompressor"))]
|
||||
async fn decompress(&self, zc_packet: &mut ZCPacket) -> Result<(), Error> {
|
||||
let pm_header = zc_packet.peer_manager_header().unwrap();
|
||||
if !pm_header.is_compressed() {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use dashmap::DashMap;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::cell::UnsafeCell;
|
||||
use std::fmt;
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::proto::common::{NatType, StunInfo};
|
||||
use anyhow::Context;
|
||||
use chrono::Local;
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use rand::seq::IteratorRandom;
|
||||
use socket2::{SockAddr, SockRef};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
@@ -11,7 +11,7 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
|
||||
use crate::{
|
||||
common::{
|
||||
@@ -50,7 +50,6 @@ use url::Host;
|
||||
|
||||
pub const DIRECT_CONNECTOR_SERVICE_ID: u32 = 1;
|
||||
pub const DIRECT_CONNECTOR_BLACKLIST_TIMEOUT_SEC: u64 = 300;
|
||||
const MAX_IPV6_HOLE_PUNCH_CONNECTOR_ADDRS: usize = 16;
|
||||
|
||||
static TESTING: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
@@ -85,56 +84,6 @@ fn is_usable_public_ipv6_candidate_with_mode(
|
||||
&& !ip.is_multicast()))
|
||||
}
|
||||
|
||||
fn push_ipv6_hole_punch_candidate(
|
||||
candidates: &mut Vec<Ipv6Addr>,
|
||||
ip: Ipv6Addr,
|
||||
global_ctx: &ArcGlobalCtx,
|
||||
limit: usize,
|
||||
) {
|
||||
if candidates.len() >= limit
|
||||
|| !is_usable_public_ipv6_candidate(&ip, global_ctx)
|
||||
|| candidates.contains(&ip)
|
||||
{
|
||||
return;
|
||||
}
|
||||
candidates.push(ip);
|
||||
}
|
||||
|
||||
async fn collect_ipv6_hole_punch_candidates(global_ctx: &ArcGlobalCtx) -> Vec<Ipv6Addr> {
|
||||
let mut candidates = Vec::new();
|
||||
for ip in global_ctx
|
||||
.get_stun_info_collector()
|
||||
.get_stun_info()
|
||||
.public_ip
|
||||
.iter()
|
||||
.filter_map(|ip| ip.parse::<Ipv6Addr>().ok())
|
||||
{
|
||||
push_ipv6_hole_punch_candidate(
|
||||
&mut candidates,
|
||||
ip,
|
||||
global_ctx,
|
||||
MAX_IPV6_HOLE_PUNCH_CONNECTOR_ADDRS,
|
||||
);
|
||||
}
|
||||
|
||||
let ip_list = global_ctx.get_ip_collector().collect_ip_addrs().await;
|
||||
for ip in ip_list
|
||||
.interface_ipv6s
|
||||
.iter()
|
||||
.chain(ip_list.public_ipv6.iter())
|
||||
.map(|ip| Ipv6Addr::from(*ip))
|
||||
{
|
||||
push_ipv6_hole_punch_candidate(
|
||||
&mut candidates,
|
||||
ip,
|
||||
global_ctx,
|
||||
MAX_IPV6_HOLE_PUNCH_CONNECTOR_ADDRS,
|
||||
);
|
||||
}
|
||||
|
||||
candidates
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait PeerManagerForDirectConnector {
|
||||
async fn list_peers(&self) -> Vec<PeerId>;
|
||||
@@ -204,8 +153,7 @@ impl DirectConnectorManagerData {
|
||||
async fn remote_send_udp_hole_punch_packet(
|
||||
&self,
|
||||
dst_peer_id: PeerId,
|
||||
connector_addrs: Vec<SocketAddr>,
|
||||
preferred_src_ipv6: Option<Ipv6Addr>,
|
||||
connector_addr: SocketAddr,
|
||||
remote_url: &url::Url,
|
||||
) -> Result<(), Error> {
|
||||
if !matches_scheme!(remote_url, TunnelScheme::Ip(IpScheme::Udp)) {
|
||||
@@ -236,17 +184,15 @@ impl DirectConnectorManagerData {
|
||||
.send_udp_hole_punch_packet(
|
||||
BaseController::default(),
|
||||
SendUdpHolePunchPacketRequest {
|
||||
connector_addr: connector_addrs.first().copied().map(Into::into),
|
||||
listener_port: listener_port as u32,
|
||||
preferred_src_ipv6: preferred_src_ipv6.map(Into::into),
|
||||
connector_addrs: connector_addrs.into_iter().map(Into::into).collect(),
|
||||
connector_addr: Some(connector_addr.into()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"do rpc, send udp hole punch packet to peer {} at {} with preferred source {:?}",
|
||||
dst_peer_id, remote_url, preferred_src_ipv6
|
||||
"do rpc, send udp hole punch packet to peer {} at {}",
|
||||
dst_peer_id, remote_url
|
||||
)
|
||||
})?;
|
||||
|
||||
@@ -263,41 +209,23 @@ impl DirectConnectorManagerData {
|
||||
.await
|
||||
.with_context(|| format!("failed to bind local socket for {}", remote_url))?,
|
||||
);
|
||||
let connector_ips = collect_ipv6_hole_punch_candidates(&self.global_ctx).await;
|
||||
let connector_ip = self
|
||||
.global_ctx
|
||||
.get_stun_info_collector()
|
||||
.get_stun_info()
|
||||
.public_ip
|
||||
.iter()
|
||||
.filter_map(|ip| ip.parse::<Ipv6Addr>().ok())
|
||||
.find(|ip| !self.global_ctx.is_ip_easytier_managed_ipv6(ip));
|
||||
|
||||
// ask remote to send v6 hole punch packet
|
||||
// and no matter what the result is, continue to connect
|
||||
if !connector_ips.is_empty() {
|
||||
let local_port = local_socket.local_addr()?.port();
|
||||
let connector_addrs = connector_ips
|
||||
.into_iter()
|
||||
.map(|ip| SocketAddr::new(IpAddr::V6(ip), local_port))
|
||||
.collect::<Vec<_>>();
|
||||
let preferred_src_ipv6 = match remote_url.host() {
|
||||
Some(Host::Ipv6(ip)) => Some(ip),
|
||||
_ => None,
|
||||
};
|
||||
tracing::debug!(
|
||||
?connector_addrs,
|
||||
?preferred_src_ipv6,
|
||||
?remote_url,
|
||||
"request remote IPv6 hole-punch packets"
|
||||
);
|
||||
if let Err(err) = self
|
||||
.remote_send_udp_hole_punch_packet(
|
||||
dst_peer_id,
|
||||
connector_addrs,
|
||||
preferred_src_ipv6,
|
||||
remote_url,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::debug!(
|
||||
?err,
|
||||
?remote_url,
|
||||
"remote IPv6 hole-punch packet request failed"
|
||||
);
|
||||
}
|
||||
if let Some(connector_ip) = connector_ip {
|
||||
let connector_addr =
|
||||
SocketAddr::new(IpAddr::V6(connector_ip), local_socket.local_addr()?.port());
|
||||
let _ = self
|
||||
.remote_send_udp_hole_punch_packet(dst_peer_id, connector_addr, remote_url)
|
||||
.await;
|
||||
} else {
|
||||
tracing::debug!(
|
||||
?remote_url,
|
||||
@@ -339,7 +267,7 @@ impl DirectConnectorManagerData {
|
||||
.with_context(|| format!("failed to get udp port mapping for {}", remote_url))?;
|
||||
|
||||
let _ = self
|
||||
.remote_send_udp_hole_punch_packet(dst_peer_id, vec![connector_addr], None, remote_url)
|
||||
.remote_send_udp_hole_punch_packet(dst_peer_id, connector_addr, remote_url)
|
||||
.await;
|
||||
|
||||
let udp_connector = UdpTunnelConnector::new(remote_url.clone());
|
||||
@@ -890,7 +818,7 @@ mod tests {
|
||||
tunnel::{IpScheme, TunnelScheme, matches_scheme},
|
||||
};
|
||||
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
|
||||
use super::{TESTING, mapped_listener_port, resolve_mapped_listener_addrs};
|
||||
|
||||
@@ -912,27 +840,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ipv6_hole_punch_candidates_are_deduped_filtered_and_capped() {
|
||||
let global_ctx = get_mock_global_ctx();
|
||||
let managed_ipv6: cidr::Ipv6Inet = "2001:db8::2/128".parse().unwrap();
|
||||
global_ctx.set_public_ipv6_routes(BTreeSet::from([managed_ipv6]));
|
||||
|
||||
let first: Ipv6Addr = "2001:db8::1".parse().unwrap();
|
||||
let managed = managed_ipv6.address();
|
||||
let second: Ipv6Addr = "2001:db8::3".parse().unwrap();
|
||||
let third: Ipv6Addr = "2001:db8::4".parse().unwrap();
|
||||
let mut candidates = Vec::new();
|
||||
|
||||
super::push_ipv6_hole_punch_candidate(&mut candidates, first, &global_ctx, 2);
|
||||
super::push_ipv6_hole_punch_candidate(&mut candidates, first, &global_ctx, 2);
|
||||
super::push_ipv6_hole_punch_candidate(&mut candidates, managed, &global_ctx, 2);
|
||||
super::push_ipv6_hole_punch_candidate(&mut candidates, second, &global_ctx, 2);
|
||||
super::push_ipv6_hole_punch_candidate(&mut candidates, third, &global_ctx, 2);
|
||||
|
||||
assert_eq!(candidates, vec![first, second]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn udp_ipv6_url_matches_hole_punch_branch_condition() {
|
||||
let remote_url: url::Url = "udp://[2001:db8::1]:11010".parse().unwrap();
|
||||
|
||||
@@ -6,7 +6,7 @@ use std::{
|
||||
};
|
||||
|
||||
use dashmap::DashSet;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use tokio::{sync::mpsc, task::JoinSet, time::timeout};
|
||||
|
||||
use crate::{
|
||||
|
||||
@@ -70,6 +70,8 @@ async fn set_bind_addr_for_peer_connector(
|
||||
let ips = global_ctx.get_ip_collector().collect_ip_addrs().await;
|
||||
if is_ipv4 {
|
||||
let mut bind_addrs = vec![];
|
||||
// Always include loopback so localhost connections work
|
||||
bind_addrs.push(std::net::SocketAddr::from(([127, 0, 0, 1], 0)));
|
||||
for ipv4 in ips.interface_ipv4s {
|
||||
let socket_addr = SocketAddrV4::new(ipv4.into(), 0).into();
|
||||
bind_addrs.push(socket_addr);
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
};
|
||||
|
||||
use anyhow::{Context, Error};
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use rand::Rng as _;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
};
|
||||
|
||||
use anyhow::Context;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::{
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use dashmap::{DashMap, DashSet};
|
||||
use guarden::defer;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use rand::seq::SliceRandom as _;
|
||||
use tokio::{net::UdpSocket, sync::Mutex, task::JoinSet};
|
||||
use tracing::{Instrument, Level, instrument};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use anyhow::Context;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use tokio::net::UdpSocket;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@ use both_easy_sym::{PunchBothEasySymHoleClient, PunchBothEasySymHoleServer};
|
||||
use common::{PunchHoleServerCommon, UdpNatType, UdpPunchClientMethod};
|
||||
use cone::{PunchConeHoleClient, PunchConeHoleServer};
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use once_cell::sync::Lazy;
|
||||
use quanta::Instant;
|
||||
use sym_to_cone::{PunchSymToConeHoleClient, PunchSymToConeHoleServer};
|
||||
use tokio::{sync::Mutex, task::JoinHandle};
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use std::{
|
||||
|
||||
use anyhow::Context;
|
||||
use guarden::defer;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use rand::{Rng, seq::SliceRandom};
|
||||
use tokio::{net::UdpSocket, sync::RwLock};
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
use easytier::core;
|
||||
|
||||
#[cfg(all(
|
||||
feature = "hotpath-alloc",
|
||||
any(feature = "jemalloc", feature = "mimalloc")
|
||||
))]
|
||||
compile_error!("feature `hotpath-alloc` cannot be enabled together with `jemalloc` or `mimalloc`");
|
||||
|
||||
#[cfg(all(feature = "mimalloc", not(feature = "jemalloc")))]
|
||||
use mimalloc::MiMalloc;
|
||||
|
||||
@@ -24,6 +30,16 @@ pub static malloc_conf: &[u8] = b"retain:false\0";
|
||||
rust_i18n::i18n!("locales", fallback = "en");
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
#[cfg_attr(
|
||||
all(
|
||||
feature = "hotpath",
|
||||
not(all(
|
||||
feature = "hotpath-alloc",
|
||||
any(feature = "jemalloc", feature = "mimalloc")
|
||||
))
|
||||
),
|
||||
hotpath::main
|
||||
)]
|
||||
async fn main() -> std::process::ExitCode {
|
||||
core::main().await
|
||||
}
|
||||
|
||||
@@ -7,13 +7,13 @@ use std::{
|
||||
};
|
||||
|
||||
use anyhow::Context;
|
||||
use hotpath::instant::Instant;
|
||||
use pnet::packet::{
|
||||
Packet,
|
||||
icmp::{self, IcmpCode, IcmpTypes, MutableIcmpPacket, echo_reply::MutableEchoReplyPacket},
|
||||
ip::IpNextHeaderProtocols,
|
||||
ipv4::Ipv4Packet,
|
||||
};
|
||||
use quanta::Instant;
|
||||
use socket2::Socket;
|
||||
use tokio::{
|
||||
sync::{Mutex, mpsc::UnboundedSender},
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use pnet::packet::Packet;
|
||||
use pnet::packet::ip::IpNextHeaderProtocol;
|
||||
use pnet::packet::ipv4::{self, Ipv4Flags, Ipv4Packet, MutableIpv4Packet};
|
||||
use quanta::Instant;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::time::Duration;
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ pub(crate) struct CidrSet {
|
||||
mapped_to_real: Arc<DashMap<cidr::Ipv4Cidr, cidr::Ipv4Cidr>>,
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure_all)]
|
||||
impl CidrSet {
|
||||
pub fn new(global_ctx: ArcGlobalCtx) -> Self {
|
||||
let mut ret = Self {
|
||||
|
||||
@@ -1018,7 +1018,7 @@ impl TcpProxyRpc for QuicProxyDstRpcService {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bytes::Buf;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
|
||||
/// Helper function: Create a pair of interconnected QuicSockets.
|
||||
/// Data sent by socket_a will enter socket_b's rx, and vice versa.
|
||||
|
||||
+57
-700
@@ -9,9 +9,9 @@ use std::{
|
||||
};
|
||||
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use hotpath::instant::Instant;
|
||||
#[cfg(feature = "kcp")]
|
||||
use kcp_sys::{endpoint::KcpEndpoint, stream::KcpStream};
|
||||
use quanta::Instant;
|
||||
use tokio_util::sync::{CancellationToken, DropGuard};
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
@@ -32,7 +32,7 @@ use crate::{
|
||||
tunnel::packet_def::{PacketType, ZCPacket},
|
||||
};
|
||||
use anyhow::Context;
|
||||
use dashmap::{DashMap, mapref::entry::Entry};
|
||||
use dashmap::DashMap;
|
||||
use pnet::packet::{
|
||||
Packet, ip::IpNextHeaderProtocols, ipv4::Ipv4Packet, tcp::TcpPacket, udp::UdpPacket,
|
||||
};
|
||||
@@ -164,87 +164,6 @@ struct Socks5Entry {
|
||||
|
||||
type Socks5EntrySet = Arc<DashMap<Socks5Entry, Socks5EntryData>>;
|
||||
|
||||
fn increment_entry_count(entry_count: &AtomicUsize) -> (usize, usize) {
|
||||
let old_entry_count = entry_count
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
|
||||
count.checked_add(1)
|
||||
})
|
||||
.unwrap_or_else(|count| count);
|
||||
(old_entry_count, old_entry_count.saturating_add(1))
|
||||
}
|
||||
|
||||
fn decrement_entry_count(entry_count: &AtomicUsize) -> (usize, usize) {
|
||||
decrement_entry_count_by(entry_count, 1)
|
||||
}
|
||||
|
||||
fn decrement_entry_count_by(entry_count: &AtomicUsize, delta: usize) -> (usize, usize) {
|
||||
if delta == 0 {
|
||||
let current = entry_count.load(Ordering::Relaxed);
|
||||
return (current, current);
|
||||
}
|
||||
|
||||
let old_entry_count = entry_count
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
|
||||
Some(count.saturating_sub(delta))
|
||||
})
|
||||
.unwrap_or_else(|count| count);
|
||||
(old_entry_count, old_entry_count.saturating_sub(delta))
|
||||
}
|
||||
|
||||
fn insert_entry_and_increment_count(
|
||||
entries: &Socks5EntrySet,
|
||||
entry_count: &AtomicUsize,
|
||||
entry: Socks5Entry,
|
||||
data: Socks5EntryData,
|
||||
) -> (bool, usize, usize) {
|
||||
match entries.entry(entry) {
|
||||
Entry::Occupied(mut occupied) => {
|
||||
occupied.insert(data);
|
||||
let current = entry_count.load(Ordering::Relaxed);
|
||||
(true, current, current)
|
||||
}
|
||||
Entry::Vacant(vacant) => {
|
||||
// Keep the count update inside the VacantEntry shard lock so bulk clear
|
||||
// cannot observe the inserted entry before its count is reserved.
|
||||
let (old_entry_count, new_entry_count) = increment_entry_count(entry_count);
|
||||
vacant.insert(data);
|
||||
(false, old_entry_count, new_entry_count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_insert_entry_and_increment_count(
|
||||
entries: &Socks5EntrySet,
|
||||
entry_count: &AtomicUsize,
|
||||
entry: Socks5Entry,
|
||||
data: Socks5EntryData,
|
||||
) -> bool {
|
||||
match entries.entry(entry) {
|
||||
Entry::Occupied(_) => false,
|
||||
Entry::Vacant(vacant) => {
|
||||
// See insert_entry_and_increment_count for why the count is reserved first.
|
||||
increment_entry_count(entry_count);
|
||||
vacant.insert(data);
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_entry_and_decrement_count(
|
||||
entries: &Socks5EntrySet,
|
||||
entry_count: &AtomicUsize,
|
||||
entry: &Socks5Entry,
|
||||
) -> (bool, usize, usize) {
|
||||
let removed = entries.remove(entry).is_some();
|
||||
let (old_entry_count, new_entry_count) = if removed {
|
||||
decrement_entry_count(entry_count)
|
||||
} else {
|
||||
let current = entry_count.load(Ordering::Relaxed);
|
||||
(current, current)
|
||||
};
|
||||
(removed, old_entry_count, new_entry_count)
|
||||
}
|
||||
|
||||
struct SmolTcpConnector {
|
||||
net: Arc<Net>,
|
||||
entries: Socks5EntrySet,
|
||||
@@ -271,20 +190,9 @@ impl AsyncTcpConnector for SmolTcpConnector {
|
||||
entry_type: TCP_ENTRY,
|
||||
};
|
||||
*self.current_entry.lock().unwrap() = Some(entry.clone());
|
||||
let (replaced, old_entry_count, new_entry_count) = insert_entry_and_increment_count(
|
||||
&self.entries,
|
||||
&self.entry_count,
|
||||
entry.clone(),
|
||||
Socks5EntryData::Tcp(tmp_listener),
|
||||
);
|
||||
tracing::trace!(
|
||||
?entry,
|
||||
replaced,
|
||||
old_entry_count,
|
||||
new_entry_count,
|
||||
entries_len = self.entries.len(),
|
||||
"socks5 inserted smoltcp tcp connector entry"
|
||||
);
|
||||
self.entries
|
||||
.insert(entry, Socks5EntryData::Tcp(tmp_listener));
|
||||
self.entry_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
if addr.ip() == local_addr {
|
||||
let modified_addr =
|
||||
@@ -312,16 +220,8 @@ impl Drop for SmolTcpConnector {
|
||||
fn drop(&mut self) {
|
||||
if let Some(entry) = self.current_entry.lock().unwrap().take() {
|
||||
tracing::debug!("drop smoltcp connector entry {:?}", entry);
|
||||
let (removed, old_entry_count, new_entry_count) =
|
||||
remove_entry_and_decrement_count(&self.entries, &self.entry_count, &entry);
|
||||
tracing::trace!(
|
||||
?entry,
|
||||
removed,
|
||||
old_entry_count,
|
||||
new_entry_count,
|
||||
entries_len = self.entries.len(),
|
||||
"socks5 removed smoltcp tcp connector entry"
|
||||
);
|
||||
self.entries.remove(&entry);
|
||||
self.entry_count.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -394,26 +294,11 @@ impl AsyncTcpConnector for Socks5AutoConnector {
|
||||
addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), addr.port());
|
||||
}
|
||||
|
||||
let has_smoltcp_net = self.smoltcp_net.is_some();
|
||||
let dst_peers = if has_smoltcp_net && !addr.ip().is_loopback() {
|
||||
Some(peer_mgr_arc.get_msg_dst_peer(&addr.ip()).await.0)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if !has_smoltcp_net
|
||||
|| dst_peers.as_ref().is_some_and(Vec::is_empty)
|
||||
if self.smoltcp_net.is_none()
|
||||
|| peer_mgr_arc.get_msg_dst_peer(&addr.ip()).await.0.is_empty()
|
||||
|| addr.ip().is_loopback()
|
||||
{
|
||||
// cannot find dst in virtual network, so try connect to dst directly
|
||||
tracing::trace!(
|
||||
?addr,
|
||||
src_addr = ?self.src_addr,
|
||||
has_smoltcp_net,
|
||||
dst_peer_count = dst_peers.as_ref().map(Vec::len),
|
||||
is_loopback = addr.ip().is_loopback(),
|
||||
"socks5 auto connector falling back to kernel tcp connect"
|
||||
);
|
||||
return Ok(SocksTcpStream::Tcp(
|
||||
tcp_connect_with_timeout(addr, timeout_s).await?,
|
||||
));
|
||||
@@ -425,51 +310,25 @@ impl AsyncTcpConnector for Socks5AutoConnector {
|
||||
#[cfg(feature = "kcp")]
|
||||
let connector: Box<dyn AsyncTcpConnector<S = SocksTcpStream> + Send> =
|
||||
match (&self.kcp_endpoint, dst_allow_kcp) {
|
||||
(Some(kcp_endpoint), true) => {
|
||||
tracing::trace!(
|
||||
?addr,
|
||||
src_addr = ?self.src_addr,
|
||||
dst_peer_count = dst_peers.as_ref().map(Vec::len),
|
||||
"socks5 auto connector selected kcp"
|
||||
);
|
||||
Box::new(Socks5KcpConnector {
|
||||
kcp_endpoint: kcp_endpoint.clone(),
|
||||
peer_mgr: self.peer_mgr.clone(),
|
||||
src_addr: self.src_addr,
|
||||
})
|
||||
}
|
||||
(_, _) => {
|
||||
tracing::trace!(
|
||||
?addr,
|
||||
src_addr = ?self.src_addr,
|
||||
dst_peer_count = dst_peers.as_ref().map(Vec::len),
|
||||
dst_allow_kcp,
|
||||
has_kcp_endpoint = self.kcp_endpoint.is_some(),
|
||||
"socks5 auto connector selected smoltcp"
|
||||
);
|
||||
Box::new(SmolTcpConnector {
|
||||
net: self.smoltcp_net.clone().unwrap(),
|
||||
entries: self.entries.clone(),
|
||||
entry_count: self.entry_count.clone(),
|
||||
current_entry: std::sync::Mutex::new(None),
|
||||
})
|
||||
}
|
||||
(Some(kcp_endpoint), true) => Box::new(Socks5KcpConnector {
|
||||
kcp_endpoint: kcp_endpoint.clone(),
|
||||
peer_mgr: self.peer_mgr.clone(),
|
||||
src_addr: self.src_addr,
|
||||
}),
|
||||
(_, _) => Box::new(SmolTcpConnector {
|
||||
net: self.smoltcp_net.clone().unwrap(),
|
||||
entries: self.entries.clone(),
|
||||
entry_count: self.entry_count.clone(),
|
||||
current_entry: std::sync::Mutex::new(None),
|
||||
}),
|
||||
};
|
||||
#[cfg(not(feature = "kcp"))]
|
||||
let connector = {
|
||||
tracing::trace!(
|
||||
?addr,
|
||||
src_addr = ?self.src_addr,
|
||||
dst_peer_count = dst_peers.as_ref().map(Vec::len),
|
||||
"socks5 auto connector selected smoltcp"
|
||||
);
|
||||
Box::new(SmolTcpConnector {
|
||||
net: self.smoltcp_net.clone().unwrap(),
|
||||
entries: self.entries.clone(),
|
||||
entry_count: self.entry_count.clone(),
|
||||
current_entry: std::sync::Mutex::new(None),
|
||||
})
|
||||
};
|
||||
let connector = Box::new(SmolTcpConnector {
|
||||
net: self.smoltcp_net.clone().unwrap(),
|
||||
entries: self.entries.clone(),
|
||||
entry_count: self.entry_count.clone(),
|
||||
current_entry: std::sync::Mutex::new(None),
|
||||
});
|
||||
|
||||
let ret = connector.tcp_connect(addr, timeout_s).await;
|
||||
self.inner_connector.lock().replace(Box::new(connector));
|
||||
@@ -644,89 +503,25 @@ pub struct Socks5Server {
|
||||
#[async_trait::async_trait]
|
||||
impl PeerPacketFilter for Socks5Server {
|
||||
async fn try_process_packet_from_peer(&self, packet: ZCPacket) -> Option<ZCPacket> {
|
||||
let entry_count = self.entry_count.load(Ordering::Relaxed);
|
||||
let socks5_enabled = self.socks5_enabled.load(Ordering::Relaxed);
|
||||
if entry_count == 0 && !socks5_enabled && self.entries.is_empty() {
|
||||
if tracing::enabled!(tracing::Level::TRACE)
|
||||
&& let Some(hdr) = packet.peer_manager_header()
|
||||
&& matches!(
|
||||
hdr.packet_type,
|
||||
x if x == PacketType::Data as u8
|
||||
|| x == PacketType::DataWithKcpSrcModified as u8
|
||||
|| x == PacketType::DataWithQuicSrcModified as u8
|
||||
)
|
||||
{
|
||||
if let Some(ipv4) = Ipv4Packet::new(packet.payload()) {
|
||||
let (tcp_src_port, tcp_dst_port, tcp_flags) =
|
||||
if ipv4.get_next_level_protocol() == IpNextHeaderProtocols::Tcp {
|
||||
TcpPacket::new(ipv4.payload())
|
||||
.map(|tcp| {
|
||||
(
|
||||
Some(tcp.get_source()),
|
||||
Some(tcp.get_destination()),
|
||||
Some(tcp.get_flags()),
|
||||
)
|
||||
})
|
||||
.unwrap_or((None, None, None))
|
||||
} else {
|
||||
(None, None, None)
|
||||
};
|
||||
tracing::trace!(
|
||||
packet_type = hdr.packet_type,
|
||||
from_peer_id = hdr.from_peer_id.get(),
|
||||
to_peer_id = hdr.to_peer_id.get(),
|
||||
ipv4_src = %ipv4.get_source(),
|
||||
ipv4_dst = %ipv4.get_destination(),
|
||||
next_protocol = ?ipv4.get_next_level_protocol(),
|
||||
?tcp_src_port,
|
||||
?tcp_dst_port,
|
||||
?tcp_flags,
|
||||
entry_count,
|
||||
socks5_enabled,
|
||||
"socks5 fast gate passed packet from peer"
|
||||
);
|
||||
} else {
|
||||
tracing::trace!(
|
||||
packet_type = hdr.packet_type,
|
||||
from_peer_id = hdr.from_peer_id.get(),
|
||||
to_peer_id = hdr.to_peer_id.get(),
|
||||
entry_count,
|
||||
socks5_enabled,
|
||||
"socks5 fast gate passed non-ipv4 packet from peer"
|
||||
);
|
||||
}
|
||||
}
|
||||
if self.entry_count.load(Ordering::Relaxed) == 0
|
||||
&& !self.socks5_enabled.load(Ordering::Relaxed)
|
||||
{
|
||||
return Some(packet);
|
||||
}
|
||||
|
||||
let hdr = packet.peer_manager_header().unwrap();
|
||||
let is_modified_src_packet = matches!(
|
||||
hdr.packet_type,
|
||||
x if x == PacketType::DataWithKcpSrcModified as u8
|
||||
|| x == PacketType::DataWithQuicSrcModified as u8
|
||||
);
|
||||
if hdr.packet_type != PacketType::Data as u8 && !is_modified_src_packet {
|
||||
if hdr.packet_type != PacketType::Data as u8 {
|
||||
return Some(packet);
|
||||
}
|
||||
if is_modified_src_packet && hdr.from_peer_id != hdr.to_peer_id {
|
||||
tracing::trace!(
|
||||
packet_type = hdr.packet_type,
|
||||
from_peer_id = hdr.from_peer_id.get(),
|
||||
to_peer_id = hdr.to_peer_id.get(),
|
||||
"socks5 passed non-loopback modified-source packet from peer"
|
||||
);
|
||||
return Some(packet);
|
||||
}
|
||||
};
|
||||
|
||||
let payload_bytes = packet.payload();
|
||||
|
||||
let Some(ipv4) = Ipv4Packet::new(payload_bytes) else {
|
||||
return Some(packet);
|
||||
};
|
||||
let ipv4 = Ipv4Packet::new(payload_bytes).unwrap();
|
||||
if ipv4.get_version() != 4 {
|
||||
return Some(packet);
|
||||
}
|
||||
|
||||
let (entry_key, tcp_flags) = match ipv4.get_next_level_protocol() {
|
||||
let entry_key = match ipv4.get_next_level_protocol() {
|
||||
IpNextHeaderProtocols::Tcp => {
|
||||
let Some(tcp_packet) = TcpPacket::new(ipv4.payload()) else {
|
||||
return Some(packet);
|
||||
@@ -751,11 +546,11 @@ impl PeerPacketFilter for Socks5Server {
|
||||
entry_type: TCP_LISTEN_ENTRY,
|
||||
}
|
||||
};
|
||||
(entry, Some(tcp_packet.get_flags()))
|
||||
entry
|
||||
}
|
||||
|
||||
IpNextHeaderProtocols::Udp => {
|
||||
if IpReassembler::is_packet_fragmented(&ipv4) {
|
||||
if IpReassembler::is_packet_fragmented(&ipv4) && !self.entries.is_empty() {
|
||||
let ipv4_src: IpAddr = ipv4.get_source().into();
|
||||
// only send to smoltcp if the ipv4 src is in the entries
|
||||
let is_in_entries = self.entries.iter().any(|x| x.key().dst.ip() == ipv4_src);
|
||||
@@ -767,19 +562,7 @@ impl PeerPacketFilter for Socks5Server {
|
||||
if is_in_entries {
|
||||
// if the packet is fragmented, no matther what the payload is, need send it to both smoltcp and kernel tun. because
|
||||
// we cannot determine the udp port of the packet.
|
||||
match self.packet_sender.try_send(packet.clone()) {
|
||||
Ok(()) => tracing::trace!(
|
||||
?ipv4_src,
|
||||
entry_count = self.entry_count.load(Ordering::Relaxed),
|
||||
"socks5 delivered fragmented packet from peer to smoltcp"
|
||||
),
|
||||
Err(err) => tracing::trace!(
|
||||
?ipv4_src,
|
||||
?err,
|
||||
entry_count = self.entry_count.load(Ordering::Relaxed),
|
||||
"socks5 failed to deliver fragmented packet from peer to smoltcp"
|
||||
),
|
||||
}
|
||||
let _ = self.packet_sender.try_send(packet.clone()).ok();
|
||||
}
|
||||
return Some(packet);
|
||||
}
|
||||
@@ -787,17 +570,14 @@ impl PeerPacketFilter for Socks5Server {
|
||||
let Some(udp_packet) = UdpPacket::new(ipv4.payload()) else {
|
||||
return Some(packet);
|
||||
};
|
||||
(
|
||||
Socks5Entry {
|
||||
dst: SocketAddr::new(ipv4.get_source().into(), udp_packet.get_source()),
|
||||
src: SocketAddr::new(
|
||||
ipv4.get_destination().into(),
|
||||
udp_packet.get_destination(),
|
||||
),
|
||||
entry_type: UDP_ENTRY,
|
||||
},
|
||||
None,
|
||||
)
|
||||
Socks5Entry {
|
||||
dst: SocketAddr::new(ipv4.get_source().into(), udp_packet.get_source()),
|
||||
src: SocketAddr::new(
|
||||
ipv4.get_destination().into(),
|
||||
udp_packet.get_destination(),
|
||||
),
|
||||
entry_type: UDP_ENTRY,
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Some(packet);
|
||||
@@ -805,41 +585,12 @@ impl PeerPacketFilter for Socks5Server {
|
||||
};
|
||||
|
||||
if !self.entries.contains_key(&entry_key) {
|
||||
tracing::trace!(
|
||||
?entry_key,
|
||||
?tcp_flags,
|
||||
ipv4_src = %ipv4.get_source(),
|
||||
ipv4_dst = %ipv4.get_destination(),
|
||||
entry_count = self.entry_count.load(Ordering::Relaxed),
|
||||
socks5_enabled = self.socks5_enabled.load(Ordering::Relaxed),
|
||||
"socks5 no entry for packet from peer"
|
||||
);
|
||||
return Some(packet);
|
||||
}
|
||||
|
||||
tracing::trace!(
|
||||
?entry_key,
|
||||
?tcp_flags,
|
||||
?ipv4,
|
||||
entry_count = self.entry_count.load(Ordering::Relaxed),
|
||||
"socks5 found entry for packet from peer"
|
||||
);
|
||||
tracing::trace!(?entry_key, ?ipv4, "socks5 found entry for packet from peer");
|
||||
|
||||
match self.packet_sender.try_send(packet) {
|
||||
Ok(()) => tracing::trace!(
|
||||
?entry_key,
|
||||
?tcp_flags,
|
||||
entry_count = self.entry_count.load(Ordering::Relaxed),
|
||||
"socks5 delivered packet from peer to smoltcp"
|
||||
),
|
||||
Err(err) => tracing::trace!(
|
||||
?entry_key,
|
||||
?tcp_flags,
|
||||
?err,
|
||||
entry_count = self.entry_count.load(Ordering::Relaxed),
|
||||
"socks5 failed to deliver packet from peer to smoltcp"
|
||||
),
|
||||
}
|
||||
let _ = self.packet_sender.try_send(packet).ok();
|
||||
|
||||
None
|
||||
}
|
||||
@@ -904,22 +655,11 @@ impl Socks5Server {
|
||||
#[cfg(not(feature = "ffi-dataplane"))]
|
||||
let data_plane_active = false;
|
||||
|
||||
let active_port_forwards = cancel_tokens.len();
|
||||
let is_socks5_enabled = socks5_enabled.load(Ordering::Relaxed);
|
||||
if active_port_forwards == 0 && !is_socks5_enabled && !data_plane_active {
|
||||
let had_net = {
|
||||
let mut net_guard = net.lock().await;
|
||||
net_guard.take().is_some()
|
||||
};
|
||||
tracing::trace!(
|
||||
had_net,
|
||||
active_port_forwards,
|
||||
is_socks5_enabled,
|
||||
data_plane_active,
|
||||
entry_count = entry_count.load(Ordering::Relaxed),
|
||||
entries_len = entries.len(),
|
||||
"socks5 net update waiting for consumers"
|
||||
);
|
||||
if cancel_tokens.is_empty()
|
||||
&& !socks5_enabled.load(Ordering::Relaxed)
|
||||
&& !data_plane_active
|
||||
{
|
||||
let _ = net.lock().await.take();
|
||||
#[cfg(feature = "ffi-dataplane")]
|
||||
let _ = data_plane_net_ready.send_replace(false);
|
||||
port_forward_list_change_notifier.notified().await;
|
||||
@@ -930,34 +670,13 @@ impl Socks5Server {
|
||||
|
||||
let cur_ipv4 = global_ctx.get_ipv4();
|
||||
if prev_ipv4 != cur_ipv4 {
|
||||
let old_ipv4 = prev_ipv4;
|
||||
prev_ipv4 = cur_ipv4;
|
||||
|
||||
tracing::trace!(
|
||||
?old_ipv4,
|
||||
?cur_ipv4,
|
||||
old_entry_count = entry_count.load(Ordering::Relaxed),
|
||||
old_entries_len = entries.len(),
|
||||
udp_client_count = udp_client_map.len(),
|
||||
"socks5 net update resetting entries for ipv4 change"
|
||||
);
|
||||
let mut removed_entries = 0;
|
||||
entries.retain(|_, _| {
|
||||
removed_entries += 1;
|
||||
entry_count.fetch_sub(1, Ordering::Relaxed);
|
||||
false
|
||||
});
|
||||
let (_, new_entry_count) =
|
||||
decrement_entry_count_by(&entry_count, removed_entries);
|
||||
udp_client_map.clear();
|
||||
tracing::trace!(
|
||||
?old_ipv4,
|
||||
?cur_ipv4,
|
||||
removed_entries,
|
||||
new_entry_count,
|
||||
new_entries_len = entries.len(),
|
||||
udp_client_count = udp_client_map.len(),
|
||||
"socks5 net update reset entries complete"
|
||||
);
|
||||
|
||||
if let Some(cur_ipv4) = cur_ipv4 {
|
||||
net.lock().await.replace(Socks5ServerNet::new(
|
||||
@@ -967,23 +686,12 @@ impl Socks5Server {
|
||||
packet_recv.clone(),
|
||||
entries.clone(),
|
||||
));
|
||||
tracing::trace!(
|
||||
?cur_ipv4,
|
||||
entry_count = entry_count.load(Ordering::Relaxed),
|
||||
entries_len = entries.len(),
|
||||
"socks5 net update installed smoltcp net"
|
||||
);
|
||||
// Wake any data-plane callers waiting in
|
||||
// `wait_data_plane_net` for the smoltcp net to appear.
|
||||
#[cfg(feature = "ffi-dataplane")]
|
||||
let _ = data_plane_net_ready.send_replace(true);
|
||||
} else {
|
||||
let _ = net.lock().await.take();
|
||||
tracing::trace!(
|
||||
entry_count = entry_count.load(Ordering::Relaxed),
|
||||
entries_len = entries.len(),
|
||||
"socks5 net update removed smoltcp net"
|
||||
);
|
||||
#[cfg(feature = "ffi-dataplane")]
|
||||
let _ = data_plane_net_ready.send_replace(false);
|
||||
}
|
||||
@@ -1066,13 +774,6 @@ impl Socks5Server {
|
||||
peer_manager
|
||||
.add_packet_process_pipeline(Box::new(self.clone()))
|
||||
.await;
|
||||
tracing::trace!(
|
||||
cfg_count = cfgs.len(),
|
||||
cancel_token_count = self.cancel_tokens.len(),
|
||||
entry_count = self.entry_count.load(Ordering::Relaxed),
|
||||
entries_len = self.entries.len(),
|
||||
"socks5 peer packet pipeline registered"
|
||||
);
|
||||
|
||||
self.run_net_update_task().await;
|
||||
|
||||
@@ -1105,7 +806,6 @@ impl Socks5Server {
|
||||
connector: Box<dyn AsyncTcpConnector<S = SocksTcpStream> + Send>,
|
||||
dst_addr: SocketAddr,
|
||||
) {
|
||||
tracing::trace!(?dst_addr, "port forward: connecting to destination");
|
||||
let outgoing_socket = match connector.tcp_connect(dst_addr, 10).await {
|
||||
Ok(socket) => socket,
|
||||
Err(e) => {
|
||||
@@ -1113,7 +813,6 @@ impl Socks5Server {
|
||||
return;
|
||||
}
|
||||
};
|
||||
tracing::trace!(?dst_addr, "port forward: connected to destination");
|
||||
|
||||
let mut outgoing_socket = outgoing_socket;
|
||||
match tokio::io::copy_bidirectional(&mut incoming_socket, &mut outgoing_socket).await {
|
||||
@@ -1200,30 +899,12 @@ impl Socks5Server {
|
||||
dst_addr
|
||||
);
|
||||
|
||||
let (smoltcp_net, net_ipv4) = {
|
||||
let net_guard = net.lock().await;
|
||||
(
|
||||
net_guard.as_ref().map(|net| net.smoltcp_net.clone()),
|
||||
net_guard.as_ref().map(|net| net.ipv4_addr),
|
||||
)
|
||||
};
|
||||
tracing::trace!(
|
||||
?bind_addr,
|
||||
?dst_addr,
|
||||
client_addr = ?addr,
|
||||
has_smoltcp_net = smoltcp_net.is_some(),
|
||||
?net_ipv4,
|
||||
entry_count = entry_count.load(Ordering::Relaxed),
|
||||
entries_len = entries.len(),
|
||||
"port forward: preparing connector"
|
||||
);
|
||||
|
||||
let connector = Socks5AutoConnector {
|
||||
#[cfg(feature = "kcp")]
|
||||
kcp_endpoint: kcp_endpoint.clone(),
|
||||
peer_mgr: peer_mgr.clone(),
|
||||
entries: entries.clone(),
|
||||
smoltcp_net,
|
||||
smoltcp_net: net.lock().await.as_ref().map(|net| net.smoltcp_net.clone()),
|
||||
src_addr: addr,
|
||||
entry_count: entry_count.clone(),
|
||||
inner_connector: parking_lot::Mutex::new(None),
|
||||
@@ -1359,12 +1040,11 @@ impl Socks5Server {
|
||||
)
|
||||
};
|
||||
let socks_udp = Arc::new(sokcs_udp);
|
||||
insert_entry_and_increment_count(
|
||||
&entries,
|
||||
&entry_count,
|
||||
entries.insert(
|
||||
client_info.entry_key.clone(),
|
||||
Socks5EntryData::Udp((socks_udp.clone(), udp_client_key.clone())),
|
||||
);
|
||||
entry_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let socks = socket.clone();
|
||||
let client_addr = addr;
|
||||
@@ -1427,18 +1107,16 @@ impl Socks5Server {
|
||||
now.duration_since(client_info.last_active.load()).as_secs() < 600
|
||||
});
|
||||
udp_forward_task.retain(|k, _| udp_client_map.contains_key(k));
|
||||
let mut removed_entries = 0;
|
||||
entries.retain(|_, data| match data {
|
||||
Socks5EntryData::Udp((_, udp_client_key)) => {
|
||||
let keep = udp_client_map.contains_key(udp_client_key);
|
||||
if !keep {
|
||||
removed_entries += 1;
|
||||
entry_count.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
keep
|
||||
}
|
||||
_ => true,
|
||||
});
|
||||
decrement_entry_count_by(&entry_count, removed_entries);
|
||||
|
||||
udp_client_map.shrink_to_fit();
|
||||
udp_forward_task.shrink_to_fit();
|
||||
@@ -1450,324 +1128,3 @@ impl Socks5Server {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
|
||||
use pnet::packet::{
|
||||
MutablePacket,
|
||||
ip::IpNextHeaderProtocols,
|
||||
ipv4::{self, MutableIpv4Packet},
|
||||
tcp::{self, MutableTcpPacket, TcpFlags},
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::peers::tests::create_mock_peer_manager;
|
||||
|
||||
fn build_tcp_packet(src: SocketAddr, dst: SocketAddr) -> Vec<u8> {
|
||||
let mut buf = vec![0u8; 40];
|
||||
let src_ip = match src.ip() {
|
||||
IpAddr::V4(ip) => ip,
|
||||
IpAddr::V6(_) => panic!("test only supports ipv4"),
|
||||
};
|
||||
let dst_ip = match dst.ip() {
|
||||
IpAddr::V4(ip) => ip,
|
||||
IpAddr::V6(_) => panic!("test only supports ipv4"),
|
||||
};
|
||||
|
||||
{
|
||||
let mut ip_packet = MutableIpv4Packet::new(&mut buf).unwrap();
|
||||
ip_packet.set_version(4);
|
||||
ip_packet.set_header_length(5);
|
||||
ip_packet.set_total_length(40);
|
||||
ip_packet.set_ttl(64);
|
||||
ip_packet.set_next_level_protocol(IpNextHeaderProtocols::Tcp);
|
||||
ip_packet.set_source(src_ip);
|
||||
ip_packet.set_destination(dst_ip);
|
||||
|
||||
let mut tcp_packet = MutableTcpPacket::new(ip_packet.payload_mut()).unwrap();
|
||||
tcp_packet.set_source(src.port());
|
||||
tcp_packet.set_destination(dst.port());
|
||||
tcp_packet.set_data_offset(5);
|
||||
tcp_packet.set_flags(TcpFlags::SYN | TcpFlags::ACK);
|
||||
tcp_packet.set_window(65535);
|
||||
tcp_packet.set_checksum(tcp::ipv4_checksum(
|
||||
&tcp_packet.to_immutable(),
|
||||
&src_ip,
|
||||
&dst_ip,
|
||||
));
|
||||
|
||||
ip_packet.set_checksum(ipv4::checksum(&ip_packet.to_immutable()));
|
||||
}
|
||||
|
||||
buf
|
||||
}
|
||||
|
||||
fn build_udp_followup_fragment(src: Ipv4Addr, dst: Ipv4Addr) -> Vec<u8> {
|
||||
let mut buf = vec![0u8; 28];
|
||||
{
|
||||
let mut ip_packet = MutableIpv4Packet::new(&mut buf).unwrap();
|
||||
ip_packet.set_version(4);
|
||||
ip_packet.set_header_length(5);
|
||||
ip_packet.set_total_length(28);
|
||||
ip_packet.set_ttl(64);
|
||||
ip_packet.set_next_level_protocol(IpNextHeaderProtocols::Udp);
|
||||
ip_packet.set_fragment_offset(1);
|
||||
ip_packet.set_source(src);
|
||||
ip_packet.set_destination(dst);
|
||||
ip_packet
|
||||
.payload_mut()
|
||||
.copy_from_slice(&[0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe, 0xba, 0xbe]);
|
||||
|
||||
ip_packet.set_checksum(ipv4::checksum(&ip_packet.to_immutable()));
|
||||
}
|
||||
|
||||
buf
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socks5_consumes_modified_data_when_entry_matches() {
|
||||
let peer_manager = create_mock_peer_manager().await;
|
||||
let server = Socks5Server::new(peer_manager.get_global_ctx(), peer_manager, None);
|
||||
|
||||
let local = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 1)), 40000);
|
||||
let remote = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 3)), 22);
|
||||
let entry = Socks5Entry {
|
||||
src: local,
|
||||
dst: remote,
|
||||
entry_type: TCP_ENTRY,
|
||||
};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
insert_entry_and_increment_count(
|
||||
&server.entries,
|
||||
&server.entry_count,
|
||||
entry,
|
||||
Socks5EntryData::Tcp(listener),
|
||||
);
|
||||
|
||||
for packet_type in [
|
||||
PacketType::DataWithKcpSrcModified,
|
||||
PacketType::DataWithQuicSrcModified,
|
||||
] {
|
||||
let mut packet = ZCPacket::new_with_payload(&build_tcp_packet(remote, local));
|
||||
packet.fill_peer_manager_hdr(1, 1, packet_type as u8);
|
||||
|
||||
let result = server.try_process_packet_from_peer(packet).await;
|
||||
assert!(result.is_none());
|
||||
|
||||
let mut receiver = server.packet_recv.lock().await;
|
||||
let received = receiver.try_recv().unwrap();
|
||||
assert_eq!(
|
||||
received.peer_manager_header().unwrap().packet_type,
|
||||
packet_type as u8
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socks5_passes_through_unmatched_or_malformed_modified_data() {
|
||||
let peer_manager = create_mock_peer_manager().await;
|
||||
let server = Socks5Server::new(peer_manager.get_global_ctx(), peer_manager, None);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
insert_entry_and_increment_count(
|
||||
&server.entries,
|
||||
&server.entry_count,
|
||||
Socks5Entry {
|
||||
src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 1)), 40000),
|
||||
dst: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 3)), 22),
|
||||
entry_type: TCP_ENTRY,
|
||||
},
|
||||
Socks5EntryData::Tcp(listener),
|
||||
);
|
||||
|
||||
let unmatched_local = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 1)), 40001);
|
||||
let remote = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 3)), 22);
|
||||
let mut unmatched_packet =
|
||||
ZCPacket::new_with_payload(&build_tcp_packet(remote, unmatched_local));
|
||||
unmatched_packet.fill_peer_manager_hdr(1, 2, PacketType::DataWithKcpSrcModified as u8);
|
||||
let result = server.try_process_packet_from_peer(unmatched_packet).await;
|
||||
assert!(result.is_some());
|
||||
|
||||
let mut malformed_packet = ZCPacket::new_with_payload(&[0u8; 8]);
|
||||
malformed_packet.fill_peer_manager_hdr(1, 2, PacketType::DataWithQuicSrcModified as u8);
|
||||
let result = server.try_process_packet_from_peer(malformed_packet).await;
|
||||
assert!(result.is_some());
|
||||
|
||||
let mut receiver = server.packet_recv.lock().await;
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socks5_passes_through_non_loopback_modified_data_even_when_entry_matches() {
|
||||
let peer_manager = create_mock_peer_manager().await;
|
||||
let server = Socks5Server::new(peer_manager.get_global_ctx(), peer_manager, None);
|
||||
|
||||
let local = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 1)), 40000);
|
||||
let remote = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 3)), 22);
|
||||
let entry = Socks5Entry {
|
||||
src: local,
|
||||
dst: remote,
|
||||
entry_type: TCP_ENTRY,
|
||||
};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
insert_entry_and_increment_count(
|
||||
&server.entries,
|
||||
&server.entry_count,
|
||||
entry,
|
||||
Socks5EntryData::Tcp(listener),
|
||||
);
|
||||
|
||||
let mut packet = ZCPacket::new_with_payload(&build_tcp_packet(remote, local));
|
||||
packet.fill_peer_manager_hdr(1, 2, PacketType::DataWithKcpSrcModified as u8);
|
||||
|
||||
let result = server.try_process_packet_from_peer(packet).await;
|
||||
assert!(result.is_some());
|
||||
|
||||
let mut receiver = server.packet_recv.lock().await;
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socks5_mirrors_fragmented_udp_even_when_entry_count_is_stale_zero() {
|
||||
let peer_manager = create_mock_peer_manager().await;
|
||||
let server = Socks5Server::new(peer_manager.get_global_ctx(), peer_manager, None);
|
||||
|
||||
let local = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 1)), 40000);
|
||||
let remote = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 144, 144, 3)), 53);
|
||||
let udp_socket = Arc::new(tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap());
|
||||
server.entries.insert(
|
||||
Socks5Entry {
|
||||
src: local,
|
||||
dst: remote,
|
||||
entry_type: UDP_ENTRY,
|
||||
},
|
||||
Socks5EntryData::Udp((
|
||||
Arc::new(SocksUdpSocket::UdpSocket(udp_socket)),
|
||||
UdpClientKey {
|
||||
client_addr: local,
|
||||
dst_addr: remote,
|
||||
},
|
||||
)),
|
||||
);
|
||||
assert_eq!(server.entry_count.load(Ordering::Relaxed), 0);
|
||||
|
||||
let mut packet = ZCPacket::new_with_payload(&build_udp_followup_fragment(
|
||||
match remote.ip() {
|
||||
IpAddr::V4(ip) => ip,
|
||||
IpAddr::V6(_) => unreachable!(),
|
||||
},
|
||||
match local.ip() {
|
||||
IpAddr::V4(ip) => ip,
|
||||
IpAddr::V6(_) => unreachable!(),
|
||||
},
|
||||
));
|
||||
packet.fill_peer_manager_hdr(1, 2, PacketType::Data as u8);
|
||||
|
||||
let result = server.try_process_packet_from_peer(packet).await;
|
||||
assert!(result.is_some());
|
||||
|
||||
let mut receiver = server.packet_recv.lock().await;
|
||||
let received = receiver.try_recv().unwrap();
|
||||
assert_eq!(
|
||||
received.peer_manager_header().unwrap().packet_type,
|
||||
PacketType::Data as u8
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrement_entry_count_does_not_underflow() {
|
||||
let entry_count = AtomicUsize::new(0);
|
||||
|
||||
let (old_entry_count, new_entry_count) = decrement_entry_count(&entry_count);
|
||||
|
||||
assert_eq!(old_entry_count, 0);
|
||||
assert_eq!(new_entry_count, 0);
|
||||
assert_eq!(entry_count.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removing_missing_entry_does_not_decrement_entry_count() {
|
||||
let entries = Arc::new(DashMap::new());
|
||||
let entry_count = AtomicUsize::new(1);
|
||||
let entry = Socks5Entry {
|
||||
src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 42, 0, 2)), 40000),
|
||||
dst: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 42, 0, 1)), 22),
|
||||
entry_type: TCP_ENTRY,
|
||||
};
|
||||
|
||||
let (removed, old_entry_count, new_entry_count) =
|
||||
remove_entry_and_decrement_count(&entries, &entry_count, &entry);
|
||||
|
||||
assert!(!removed);
|
||||
assert_eq!(old_entry_count, 1);
|
||||
assert_eq!(new_entry_count, 1);
|
||||
assert_eq!(entry_count.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removing_present_entry_decrements_entry_count_once() {
|
||||
let entries = Arc::new(DashMap::new());
|
||||
let entry_count = AtomicUsize::new(0);
|
||||
let entry = Socks5Entry {
|
||||
src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 42, 0, 2)), 40000),
|
||||
dst: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 42, 0, 1)), 22),
|
||||
entry_type: TCP_ENTRY,
|
||||
};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
insert_entry_and_increment_count(
|
||||
&entries,
|
||||
&entry_count,
|
||||
entry.clone(),
|
||||
Socks5EntryData::Tcp(listener),
|
||||
);
|
||||
|
||||
let (removed, old_entry_count, new_entry_count) =
|
||||
remove_entry_and_decrement_count(&entries, &entry_count, &entry);
|
||||
let (removed_again, old_entry_count_again, new_entry_count_again) =
|
||||
remove_entry_and_decrement_count(&entries, &entry_count, &entry);
|
||||
|
||||
assert!(removed);
|
||||
assert_eq!(old_entry_count, 1);
|
||||
assert_eq!(new_entry_count, 0);
|
||||
assert!(!removed_again);
|
||||
assert_eq!(old_entry_count_again, 0);
|
||||
assert_eq!(new_entry_count_again, 0);
|
||||
assert_eq!(entry_count.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacing_present_entry_does_not_increment_entry_count() {
|
||||
let entries = Arc::new(DashMap::new());
|
||||
let entry_count = AtomicUsize::new(0);
|
||||
let entry = Socks5Entry {
|
||||
src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 42, 0, 2)), 40000),
|
||||
dst: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 42, 0, 1)), 22),
|
||||
entry_type: TCP_ENTRY,
|
||||
};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let replacement = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
let (replaced, old_entry_count, new_entry_count) = insert_entry_and_increment_count(
|
||||
&entries,
|
||||
&entry_count,
|
||||
entry.clone(),
|
||||
Socks5EntryData::Tcp(listener),
|
||||
);
|
||||
let (replaced_again, old_entry_count_again, new_entry_count_again) =
|
||||
insert_entry_and_increment_count(
|
||||
&entries,
|
||||
&entry_count,
|
||||
entry,
|
||||
Socks5EntryData::Tcp(replacement),
|
||||
);
|
||||
|
||||
assert!(!replaced);
|
||||
assert_eq!(old_entry_count, 0);
|
||||
assert_eq!(new_entry_count, 1);
|
||||
assert!(replaced_again);
|
||||
assert_eq!(old_entry_count_again, 1);
|
||||
assert_eq!(new_entry_count_again, 1);
|
||||
assert_eq!(entry_count.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,8 @@ use std::{
|
||||
};
|
||||
|
||||
use anyhow::Context as _;
|
||||
use quanta::Instant;
|
||||
use dashmap::mapref::entry::Entry;
|
||||
use hotpath::instant::Instant;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
|
||||
use crate::{common::error::Error, gateway::fast_socks5::server::AsyncTcpConnector};
|
||||
@@ -34,7 +35,6 @@ use crate::{common::error::Error, gateway::fast_socks5::server::AsyncTcpConnecto
|
||||
use super::{
|
||||
Socks5AutoConnector, Socks5Entry, Socks5EntryData, Socks5EntrySet, Socks5Server,
|
||||
SocksTcpStream, SocksUdpSocket, TCP_ENTRY, TCP_LISTEN_ENTRY, UDP_ENTRY, UdpClientKey,
|
||||
decrement_entry_count, insert_entry_and_increment_count, try_insert_entry_and_increment_count,
|
||||
};
|
||||
use crate::gateway::tokio_smoltcp::{Net, TcpListener};
|
||||
|
||||
@@ -59,12 +59,12 @@ impl OwnedRouteEntry {
|
||||
entry_count: Arc<AtomicUsize>,
|
||||
entry: Socks5Entry,
|
||||
) -> Self {
|
||||
insert_entry_and_increment_count(
|
||||
&entries,
|
||||
&entry_count,
|
||||
entry.clone(),
|
||||
Socks5EntryData::DataPlaneRoute,
|
||||
);
|
||||
if entries
|
||||
.insert(entry.clone(), Socks5EntryData::DataPlaneRoute)
|
||||
.is_none()
|
||||
{
|
||||
entry_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Self {
|
||||
entries,
|
||||
entry_count,
|
||||
@@ -78,13 +78,12 @@ impl OwnedRouteEntry {
|
||||
entry_count: Arc<AtomicUsize>,
|
||||
entry: Socks5Entry,
|
||||
) -> Option<Self> {
|
||||
if !try_insert_entry_and_increment_count(
|
||||
&entries,
|
||||
&entry_count,
|
||||
entry.clone(),
|
||||
Socks5EntryData::DataPlaneRoute,
|
||||
) {
|
||||
return None;
|
||||
match entries.entry(entry.clone()) {
|
||||
Entry::Occupied(_) => return None,
|
||||
Entry::Vacant(vacant) => {
|
||||
vacant.insert(Socks5EntryData::DataPlaneRoute);
|
||||
entry_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
Some(Self {
|
||||
entries,
|
||||
@@ -97,7 +96,7 @@ impl OwnedRouteEntry {
|
||||
impl Drop for OwnedRouteEntry {
|
||||
fn drop(&mut self) {
|
||||
if self.entries.remove(&self.entry).is_some() {
|
||||
decrement_entry_count(&self.entry_count);
|
||||
self.entry_count.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -225,18 +224,16 @@ impl DataPlaneUdpSocket {
|
||||
dst: addr,
|
||||
entry_type: UDP_ENTRY,
|
||||
};
|
||||
try_insert_entry_and_increment_count(
|
||||
&self.entries,
|
||||
&self.entry_count,
|
||||
key,
|
||||
Socks5EntryData::Udp((
|
||||
if let Entry::Vacant(entry) = self.entries.entry(key) {
|
||||
entry.insert(Socks5EntryData::Udp((
|
||||
self.socket.clone(),
|
||||
UdpClientKey {
|
||||
client_addr: self.local_addr,
|
||||
dst_addr: addr,
|
||||
},
|
||||
)),
|
||||
);
|
||||
)));
|
||||
self.entry_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.socket.send_to(buf, addr).await
|
||||
}
|
||||
|
||||
@@ -247,15 +244,13 @@ impl DataPlaneUdpSocket {
|
||||
|
||||
impl Drop for DataPlaneUdpSocket {
|
||||
fn drop(&mut self) {
|
||||
let mut removed_entries = 0;
|
||||
self.entries.retain(|_, data| match data {
|
||||
Socks5EntryData::Udp((socket, _)) if Arc::ptr_eq(socket, &self.socket) => {
|
||||
removed_entries += 1;
|
||||
self.entry_count.fetch_sub(1, Ordering::Relaxed);
|
||||
false
|
||||
}
|
||||
_ => true,
|
||||
});
|
||||
super::decrement_entry_count_by(&self.entry_count, removed_entries);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,12 +3,12 @@ use cidr::Ipv4Inet;
|
||||
use core::panic;
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use pnet::packet::MutablePacket;
|
||||
use pnet::packet::Packet;
|
||||
use pnet::packet::ip::IpNextHeaderProtocols;
|
||||
use pnet::packet::ipv4::{Ipv4Packet, MutableIpv4Packet};
|
||||
use pnet::packet::tcp::{MutableTcpPacket, TcpPacket, ipv4_checksum};
|
||||
use quanta::Instant;
|
||||
use socket2::{SockRef, TcpKeepalive};
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU16};
|
||||
|
||||
@@ -8,13 +8,13 @@ use bytes::{BufMut, BytesMut};
|
||||
use cidr::Ipv4Inet;
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use pnet::packet::{
|
||||
Packet,
|
||||
ip::IpNextHeaderProtocols,
|
||||
ipv4::Ipv4Packet,
|
||||
udp::{self, MutableUdpPacket},
|
||||
};
|
||||
use quanta::Instant;
|
||||
use tokio::sync::mpsc::{Receiver, Sender, channel, error::TrySendError};
|
||||
use tokio::{
|
||||
net::UdpSocket,
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
//! No-op stand-in for the `hotpath` macros used by this crate, selected when
|
||||
//! the `hotpath` feature is disabled.
|
||||
//!
|
||||
//! Keeping `hotpath` as an optional dependency means default builds do not pull
|
||||
//! the profiler (or any of its transitive dependencies) into the dependency
|
||||
//! graph. These macros expand to their input unchanged, mirroring `hotpath`'s
|
||||
//! own disabled mode so call sites compile identically with or without the
|
||||
//! feature.
|
||||
//!
|
||||
//! The macros are `#[macro_export]`-ed so that `lib.rs`' `extern crate self as
|
||||
//! hotpath` alias exposes them through the same `hotpath::...` paths used when
|
||||
//! the feature is enabled.
|
||||
|
||||
/// No-op mirroring `hotpath::channel!`: returns the channel expression
|
||||
/// unchanged (dropping any optional trailing `label`/`log`/`capacity` args).
|
||||
#[doc(hidden)]
|
||||
#[macro_export]
|
||||
macro_rules! channel {
|
||||
($expr:expr $(, $($rest:tt)*)?) => {
|
||||
$expr
|
||||
};
|
||||
}
|
||||
|
||||
/// No-op mirroring `hotpath::mutex!`: returns the expression unchanged.
|
||||
#[doc(hidden)]
|
||||
#[macro_export]
|
||||
macro_rules! mutex {
|
||||
($expr:expr $(, $($rest:tt)*)?) => {
|
||||
$expr
|
||||
};
|
||||
}
|
||||
|
||||
/// No-op mirroring `hotpath::rw_lock!`: returns the expression unchanged.
|
||||
#[doc(hidden)]
|
||||
#[macro_export]
|
||||
macro_rules! rw_lock {
|
||||
($expr:expr $(, $($rest:tt)*)?) => {
|
||||
$expr
|
||||
};
|
||||
}
|
||||
@@ -483,22 +483,18 @@ impl InstanceConfigPatcher {
|
||||
}
|
||||
let global_ctx = weak_upgrade(&self.global_ctx)?;
|
||||
for proxy_network_patch in proxy_networks {
|
||||
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||
tracing::warn!("Proxy network cidr is None, skipping.");
|
||||
continue;
|
||||
};
|
||||
let mapped_cidr: Option<cidr::Ipv4Cidr> =
|
||||
proxy_network_patch.mapped_cidr.map(|s| s.into());
|
||||
match ConfigPatchAction::try_from(proxy_network_patch.action) {
|
||||
Ok(ConfigPatchAction::Add) => {
|
||||
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||
tracing::warn!("Proxy network cidr is None, skipping add.");
|
||||
continue;
|
||||
};
|
||||
let mapped_cidr: Option<cidr::Ipv4Cidr> =
|
||||
proxy_network_patch.mapped_cidr.map(|s| s.into());
|
||||
tracing::info!("Proxy network added: {}", cidr);
|
||||
global_ctx.config.add_proxy_cidr(cidr, mapped_cidr)?;
|
||||
}
|
||||
Ok(ConfigPatchAction::Remove) => {
|
||||
let Some(cidr) = proxy_network_patch.cidr.map(|c| c.into()) else {
|
||||
tracing::warn!("Proxy network cidr is None, skipping remove.");
|
||||
continue;
|
||||
};
|
||||
tracing::info!("Proxy network removed: {}", cidr);
|
||||
global_ctx.config.remove_proxy_cidr(cidr);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::sync::{Arc, Weak};
|
||||
|
||||
use crate::common::global_ctx::{ArcGlobalCtx, GlobalCtxEvent};
|
||||
use crate::peers::peer_manager::PeerManager;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
/// ProxyCidrsMonitor monitors changes in proxy CIDRs from peer routes
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::{
|
||||
};
|
||||
|
||||
use byteorder::WriteBytesExt as _;
|
||||
use bytes::{BufMut, BytesMut};
|
||||
use bytes::{Buf, BufMut, BytesMut};
|
||||
use cidr::{Ipv4Inet, Ipv6Inet};
|
||||
use futures::{SinkExt, Stream, StreamExt, lock::BiLock, ready};
|
||||
use pin_project_lite::pin_project;
|
||||
@@ -180,12 +180,13 @@ impl ZCPacketToBytes for TunZCPacketToBytes {
|
||||
assert!(payload_offset >= 4);
|
||||
|
||||
let ret = if self.has_packet_info {
|
||||
let mut inner = inner.split_off(payload_offset - 4);
|
||||
inner.advance(payload_offset - 4);
|
||||
let proto = infer_proto(&inner[4..]);
|
||||
self.fill_packet_info(&mut inner[0..4], proto)?;
|
||||
inner
|
||||
} else {
|
||||
inner.split_off(payload_offset)
|
||||
inner.advance(payload_offset);
|
||||
inner
|
||||
};
|
||||
|
||||
tracing::debug!(?ret, ?payload_offset, "convert zc packet to tun packet");
|
||||
|
||||
+19
-151
@@ -626,47 +626,6 @@ pub type NetworkingMethod = crate::proto::api::manage::NetworkingMethod;
|
||||
pub type NetworkConfig = crate::proto::api::manage::NetworkConfig;
|
||||
|
||||
impl NetworkConfig {
|
||||
fn parse_peer(peer: &manage::NetworkPeerConfig) -> Result<Option<PeerConfig>, anyhow::Error> {
|
||||
let uri = peer.uri.trim();
|
||||
if uri.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(Some(PeerConfig {
|
||||
uri: uri
|
||||
.parse()
|
||||
.with_context(|| format!("failed to parse peer uri: {}", uri))?,
|
||||
peer_public_key: peer.peer_public_key.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_peers(peers: &[manage::NetworkPeerConfig]) -> Result<Vec<PeerConfig>, anyhow::Error> {
|
||||
let mut ret = Vec::new();
|
||||
for peer in peers {
|
||||
if let Some(peer) = Self::parse_peer(peer)? {
|
||||
ret.push(peer);
|
||||
}
|
||||
}
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
fn parse_peer_urls(peer_urls: &[String]) -> Result<Vec<PeerConfig>, anyhow::Error> {
|
||||
let mut peers = vec![];
|
||||
for peer_url in peer_urls.iter() {
|
||||
let peer_url = peer_url.trim();
|
||||
if peer_url.is_empty() {
|
||||
continue;
|
||||
}
|
||||
peers.push(PeerConfig {
|
||||
uri: peer_url
|
||||
.parse()
|
||||
.with_context(|| format!("failed to parse peer uri: {}", peer_url))?,
|
||||
peer_public_key: None,
|
||||
});
|
||||
}
|
||||
Ok(peers)
|
||||
}
|
||||
|
||||
pub fn gen_config(&self) -> Result<TomlConfigLoader, anyhow::Error> {
|
||||
let cfg = TomlConfigLoader::default();
|
||||
cfg.set_id(
|
||||
@@ -722,23 +681,26 @@ impl NetworkConfig {
|
||||
.unwrap_or_default()
|
||||
{
|
||||
NetworkingMethod::PublicServer => {
|
||||
let peers = Self::parse_peers(&self.peers)?;
|
||||
if peers.is_empty() {
|
||||
let public_server_url = self.public_server_url.clone().unwrap_or_default();
|
||||
cfg.set_peers(vec![PeerConfig {
|
||||
uri: public_server_url.parse().with_context(|| {
|
||||
format!("failed to parse public server uri: {}", public_server_url)
|
||||
})?,
|
||||
peer_public_key: None,
|
||||
}]);
|
||||
} else {
|
||||
cfg.set_peers(peers);
|
||||
}
|
||||
let public_server_url = self.public_server_url.clone().unwrap_or_default();
|
||||
cfg.set_peers(vec![PeerConfig {
|
||||
uri: public_server_url.parse().with_context(|| {
|
||||
format!("failed to parse public server uri: {}", public_server_url)
|
||||
})?,
|
||||
peer_public_key: None,
|
||||
}]);
|
||||
}
|
||||
NetworkingMethod::Manual => {
|
||||
let mut peers = Self::parse_peers(&self.peers)?;
|
||||
if peers.is_empty() {
|
||||
peers = Self::parse_peer_urls(&self.peer_urls)?;
|
||||
let mut peers = vec![];
|
||||
for peer_url in self.peer_urls.iter() {
|
||||
if peer_url.is_empty() {
|
||||
continue;
|
||||
}
|
||||
peers.push(PeerConfig {
|
||||
uri: peer_url
|
||||
.parse()
|
||||
.with_context(|| format!("failed to parse peer uri: {}", peer_url))?,
|
||||
peer_public_key: None,
|
||||
});
|
||||
}
|
||||
if !peers.is_empty() {
|
||||
cfg.set_peers(peers);
|
||||
@@ -1082,13 +1044,6 @@ impl NetworkConfig {
|
||||
result.networking_method = Some(NetworkingMethod::Manual as i32);
|
||||
if !peers.is_empty() {
|
||||
result.peer_urls = peers.iter().map(|p| p.uri.to_string()).collect();
|
||||
result.peers = peers
|
||||
.iter()
|
||||
.map(|p| manage::NetworkPeerConfig {
|
||||
uri: p.uri.to_string(),
|
||||
peer_public_key: p.peer_public_key.clone(),
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
|
||||
result.listener_urls = config
|
||||
@@ -1161,7 +1116,6 @@ impl NetworkConfig {
|
||||
.get_credential_file()
|
||||
.map(|path| path.to_string_lossy().into_owned());
|
||||
let flags = config.get_flags();
|
||||
let default_flags = default_config.get_flags();
|
||||
result.latency_first = Some(flags.latency_first);
|
||||
result.dev_name = Some(flags.dev_name.clone());
|
||||
result.use_smoltcp = Some(flags.use_smoltcp);
|
||||
@@ -1190,11 +1144,6 @@ impl NetworkConfig {
|
||||
result.disable_sym_hole_punching = Some(flags.disable_sym_hole_punching);
|
||||
result.enable_magic_dns = Some(flags.accept_dns);
|
||||
result.mtu = Some(flags.mtu as i32);
|
||||
result.data_compress_algo = (flags.data_compress_algo != default_flags.data_compress_algo)
|
||||
.then_some(flags.data_compress_algo);
|
||||
result.encryption_algorithm = (flags.encryption_algorithm
|
||||
!= default_flags.encryption_algorithm)
|
||||
.then_some(flags.encryption_algorithm.clone());
|
||||
result.instance_recv_bps_limit =
|
||||
(flags.instance_recv_bps_limit != u64::MAX).then_some(flags.instance_recv_bps_limit);
|
||||
result.enable_private_mode = Some(flags.private_mode);
|
||||
@@ -1224,7 +1173,7 @@ impl NetworkConfig {
|
||||
mod tests {
|
||||
use crate::{
|
||||
common::config::{ConfigLoader, process_secure_mode_cfg},
|
||||
proto::common::{CompressionAlgoPb, SecureModeConfig},
|
||||
proto::common::SecureModeConfig,
|
||||
};
|
||||
use base64::prelude::{BASE64_STANDARD, Engine as _};
|
||||
use rand::Rng;
|
||||
@@ -1287,54 +1236,6 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_network_config_conversion_preserves_peer_public_key() -> Result<(), anyhow::Error> {
|
||||
let peer_url = "tcp://1.2.3.4:11010";
|
||||
let peer_public_key = BASE64_STANDARD.encode([9u8; 32]);
|
||||
let config = gen_default_config();
|
||||
config.set_peers(vec![crate::common::config::PeerConfig {
|
||||
uri: peer_url.parse()?,
|
||||
peer_public_key: Some(peer_public_key.clone()),
|
||||
}]);
|
||||
|
||||
let network_config = super::NetworkConfig::new_from_config(&config)?;
|
||||
|
||||
assert_eq!(network_config.peer_urls, vec![peer_url.to_string()]);
|
||||
assert_eq!(network_config.peers.len(), 1);
|
||||
assert_eq!(network_config.peers[0].uri, peer_url);
|
||||
assert_eq!(
|
||||
network_config.peers[0].peer_public_key.as_deref(),
|
||||
Some(peer_public_key.as_str())
|
||||
);
|
||||
|
||||
let generated_config = network_config.gen_config()?;
|
||||
assert_eq!(generated_config.get_peers(), config.get_peers());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_config_gen_config_trims_legacy_peer_urls() -> Result<(), anyhow::Error> {
|
||||
let network_config = super::NetworkConfig {
|
||||
instance_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
dhcp: Some(true),
|
||||
networking_method: Some(crate::proto::api::manage::NetworkingMethod::Manual as i32),
|
||||
peer_urls: vec![
|
||||
" tcp://1.2.3.4:11010 ".to_string(),
|
||||
" ".to_string(),
|
||||
"\tudp://5.6.7.8:11010\n".to_string(),
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let generated_config = network_config.gen_config()?;
|
||||
let peers = generated_config.get_peers();
|
||||
|
||||
assert_eq!(peers.len(), 2);
|
||||
assert_eq!(peers[0].uri.as_str(), "tcp://1.2.3.4:11010");
|
||||
assert_eq!(peers[1].uri.as_str(), "udp://5.6.7.8:11010");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_network_config_conversion_random() -> Result<(), anyhow::Error> {
|
||||
let mut rng = rand::thread_rng();
|
||||
@@ -1633,37 +1534,4 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_network_config_conversion_preserves_runtime_algorithm_flags()
|
||||
-> Result<(), anyhow::Error> {
|
||||
let config = gen_default_config();
|
||||
let mut flags = config.get_flags();
|
||||
flags.data_compress_algo = CompressionAlgoPb::Zstd.into();
|
||||
flags.encryption_algorithm = "managed-test-algo".to_string();
|
||||
config.set_flags(flags.clone());
|
||||
|
||||
let network_config = super::NetworkConfig::new_from_config(&config)?;
|
||||
|
||||
assert_eq!(
|
||||
network_config.data_compress_algo,
|
||||
Some(CompressionAlgoPb::Zstd as i32)
|
||||
);
|
||||
assert_eq!(
|
||||
network_config.encryption_algorithm.as_deref(),
|
||||
Some("managed-test-algo")
|
||||
);
|
||||
|
||||
let generated_config = network_config.gen_config()?;
|
||||
assert_eq!(
|
||||
generated_config.get_flags().data_compress_algo,
|
||||
flags.data_compress_algo
|
||||
);
|
||||
assert_eq!(
|
||||
generated_config.get_flags().encryption_algorithm,
|
||||
flags.encryption_algorithm
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
+20
-1
@@ -5,10 +5,29 @@ use std::io;
|
||||
use clap::Command;
|
||||
use clap_complete::{Generator, Shell};
|
||||
|
||||
// When the `hotpath` feature is off, alias the current crate as `hotpath` so
|
||||
// call sites keep using `hotpath::...` paths, and provide a local no-op shim
|
||||
// for the profiling macros. This keeps `hotpath` an optional dependency: the
|
||||
// profiler is absent from the dependency graph entirely in default builds.
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
extern crate self as hotpath;
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
mod hotpath_off;
|
||||
|
||||
// When the `hotpath` feature is off, expose a local `instant` module backed by
|
||||
// `quanta::Instant` so call sites can uniformly write `use hotpath::instant::Instant;`
|
||||
// regardless of whether the feature is enabled. With the feature on, the real
|
||||
// `hotpath` crate provides the same path (also `quanta::Instant` on Linux), so
|
||||
// the two modes resolve to the identical type.
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
pub mod instant {
|
||||
pub type Instant = quanta::Instant;
|
||||
}
|
||||
|
||||
// Re-export `Instant` at the crate root so public APIs that expose it
|
||||
// (e.g. `Route::get_peer_info_last_update_time`) reference a deliberate
|
||||
// public type rather than leaking an inaccessible one.
|
||||
pub use quanta::Instant;
|
||||
pub use hotpath::instant::Instant;
|
||||
|
||||
mod arch;
|
||||
mod gateway;
|
||||
|
||||
@@ -7,11 +7,11 @@ use std::{
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use pnet::packet::ipv6::Ipv6Packet;
|
||||
use pnet::packet::{
|
||||
Packet as _, ip::IpNextHeaderProtocols, ipv4::Ipv4Packet, tcp::TcpPacket, udp::UdpPacket,
|
||||
};
|
||||
use quanta::Instant;
|
||||
|
||||
use crate::proto::acl::{AclStats, Protocol};
|
||||
use crate::tunnel::packet_def::PacketType;
|
||||
@@ -404,7 +404,7 @@ mod tests {
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
|
||||
use crate::{
|
||||
common::acl_processor::PacketInfo,
|
||||
|
||||
@@ -59,15 +59,21 @@ type BoxNicPacketFilter = Box<dyn NicPacketFilter + Send + Sync>;
|
||||
pub type PacketRecvChan = tokio::sync::mpsc::Sender<ZCPacket>;
|
||||
pub type PacketRecvChanReceiver = tokio::sync::mpsc::Receiver<ZCPacket>;
|
||||
pub fn create_packet_recv_chan() -> (PacketRecvChan, PacketRecvChanReceiver) {
|
||||
tokio::sync::mpsc::channel(128)
|
||||
hotpath::channel!(tokio::sync::mpsc::channel(128))
|
||||
}
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PacketRecvChan"))]
|
||||
pub async fn recv_packet_from_chan(
|
||||
packet_recv_chan_receiver: &mut PacketRecvChanReceiver,
|
||||
) -> Result<ZCPacket, anyhow::Error> {
|
||||
packet_recv_chan_receiver
|
||||
.recv()
|
||||
.await
|
||||
.ok_or(anyhow::anyhow!("recv_packet_from_chan failed"))
|
||||
use tokio::sync::mpsc::error::TryRecvError;
|
||||
match packet_recv_chan_receiver.try_recv() {
|
||||
Ok(pkt) => Ok(pkt),
|
||||
Err(TryRecvError::Empty) => packet_recv_chan_receiver
|
||||
.recv()
|
||||
.await
|
||||
.ok_or(anyhow::anyhow!("recv_packet_from_chan failed")),
|
||||
Err(TryRecvError::Disconnected) => Err(anyhow::anyhow!("recv_packet_from_chan failed")),
|
||||
}
|
||||
}
|
||||
|
||||
pub const PUBLIC_SERVER_HOSTNAME_PREFIX: &str = "PublicServer_";
|
||||
|
||||
@@ -207,6 +207,7 @@ impl Peer {
|
||||
.map(|conn| conn.clone())
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "Peer"))]
|
||||
pub async fn send_msg(&self, msg: ZCPacket) -> Result<(), Error> {
|
||||
let Some(conn) = self.select_conn().await else {
|
||||
return Err(Error::PeerNoConnectionError(self.peer_node_id));
|
||||
@@ -267,6 +268,12 @@ impl Peer {
|
||||
self.default_conn_id.load()
|
||||
}
|
||||
|
||||
pub fn set_batch_threshold(&self, n: u32) {
|
||||
for conn in self.conns.iter() {
|
||||
conn.value().set_batch_threshold(n);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_peer_identity_type(&self) -> Option<PeerIdentityType> {
|
||||
self.peer_identity_type.load()
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use arc_swap::ArcSwapOption;
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use futures::{StreamExt, TryFutureExt};
|
||||
use std::{
|
||||
@@ -11,6 +10,13 @@ use std::{
|
||||
},
|
||||
};
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
use hotpath::wrap::std::sync::Mutex as StdMutex;
|
||||
#[cfg(feature = "hotpath")]
|
||||
use hotpath::wrap::tokio::sync::Mutex;
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
use std::sync::Mutex as StdMutex;
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use base64::Engine as _;
|
||||
@@ -101,7 +107,7 @@ struct PeerSessionTunnelFilter {
|
||||
enabled: bool,
|
||||
my_peer_id: Arc<AtomicCell<PeerId>>,
|
||||
peer_id: Arc<AtomicCell<Option<PeerId>>>,
|
||||
session: Arc<ArcSwapOption<PeerSession>>,
|
||||
session: Arc<StdMutex<Option<Arc<PeerSession>>>>,
|
||||
}
|
||||
|
||||
impl PeerSessionTunnelFilter {
|
||||
@@ -110,7 +116,7 @@ impl PeerSessionTunnelFilter {
|
||||
enabled,
|
||||
my_peer_id: Arc::new(AtomicCell::new(PeerId::default())),
|
||||
peer_id: Arc::new(AtomicCell::new(None)),
|
||||
session: Arc::new(ArcSwapOption::empty()),
|
||||
session: Arc::new(hotpath::mutex!(std::sync::Mutex::new(None))),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +125,7 @@ impl PeerSessionTunnelFilter {
|
||||
enabled,
|
||||
my_peer_id: Arc::new(AtomicCell::new(my_peer_id)),
|
||||
peer_id: Arc::new(AtomicCell::new(None)),
|
||||
session: Arc::new(ArcSwapOption::empty()),
|
||||
session: Arc::new(hotpath::mutex!(std::sync::Mutex::new(None))),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,7 +138,7 @@ impl PeerSessionTunnelFilter {
|
||||
}
|
||||
|
||||
fn set_session(&self, session: Arc<PeerSession>) {
|
||||
self.session.store(Some(session));
|
||||
*self.session.lock().unwrap() = Some(session);
|
||||
}
|
||||
|
||||
fn should_skip_encrypt(&self, hdr: &crate::tunnel::packet_def::PeerManagerHeader) -> bool {
|
||||
@@ -166,15 +172,16 @@ impl TunnelFilter for PeerSessionTunnelFilter {
|
||||
return Some(data);
|
||||
};
|
||||
|
||||
let mut guard = self.session.lock().unwrap();
|
||||
let Some(session) = guard.as_mut() else {
|
||||
return Some(data);
|
||||
};
|
||||
|
||||
let my_peer_id = self.my_peer_id.load();
|
||||
if my_peer_id != hdr.from_peer_id.get() || hdr.to_peer_id.get() != peer_id {
|
||||
if my_peer_id != hdr.from_peer_id.get() {
|
||||
return Some(data);
|
||||
}
|
||||
|
||||
let session_guard = self.session.load();
|
||||
let Some(session) = session_guard.as_deref() else {
|
||||
return Some(data);
|
||||
};
|
||||
if let Err(e) = session.encrypt_payload(my_peer_id, peer_id, &mut data) {
|
||||
tracing::warn!(
|
||||
?my_peer_id,
|
||||
@@ -219,8 +226,8 @@ impl TunnelFilter for PeerSessionTunnelFilter {
|
||||
return Some(Ok(data));
|
||||
}
|
||||
|
||||
let session_guard = self.session.load();
|
||||
let Some(session) = session_guard.as_deref() else {
|
||||
let mut guard = self.session.lock().unwrap();
|
||||
let Some(session) = guard.as_mut() else {
|
||||
return Some(Ok(data));
|
||||
};
|
||||
|
||||
@@ -389,12 +396,12 @@ impl PeerConn {
|
||||
session_filter,
|
||||
noise_handshake_result: None,
|
||||
|
||||
tunnel: Arc::new(Mutex::new(
|
||||
Box::new(guard!([mut mpsc_tunnel] mpsc_tunnel.close()))
|
||||
as Box<dyn Any + Send + 'static>,
|
||||
)),
|
||||
tunnel: Arc::new(hotpath::mutex!(tokio::sync::Mutex::new(Box::new(
|
||||
guard!([mut mpsc_tunnel] mpsc_tunnel.close()),
|
||||
)
|
||||
as Box<dyn Any + Send + 'static>))),
|
||||
sink,
|
||||
recv: Mutex::new(Some(recv)),
|
||||
recv: hotpath::mutex!(tokio::sync::Mutex::new(Some(recv))),
|
||||
tunnel_info,
|
||||
|
||||
tasks: JoinSet::new(),
|
||||
@@ -444,6 +451,10 @@ impl PeerConn {
|
||||
self.conn_id
|
||||
}
|
||||
|
||||
pub fn set_batch_threshold(&self, n: u32) {
|
||||
self.sink.set_batch_threshold(n);
|
||||
}
|
||||
|
||||
pub fn set_is_hole_punched(&mut self, is_hole_punched: bool) {
|
||||
self.is_hole_punched = is_hole_punched;
|
||||
}
|
||||
@@ -1471,6 +1482,7 @@ impl PeerConn {
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerConn"))]
|
||||
pub async fn send_msg(&self, msg: ZCPacket) -> Result<(), Error> {
|
||||
Ok(self.sink.send(msg).await?)
|
||||
}
|
||||
@@ -1653,45 +1665,6 @@ pub mod tests {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_session_filter_skips_relay_packet_for_next_hop() {
|
||||
let my_peer_id = 10;
|
||||
let next_hop_peer_id = 20;
|
||||
let dst_peer_id = 30;
|
||||
let filter = PeerSessionTunnelFilter::new_with_peer(my_peer_id, true);
|
||||
filter.set_peer_id(next_hop_peer_id);
|
||||
|
||||
let session = Arc::new(PeerSession::new(
|
||||
next_hop_peer_id,
|
||||
PeerSession::new_root_key(),
|
||||
1,
|
||||
0,
|
||||
"aes-gcm".to_string(),
|
||||
"aes-gcm".to_string(),
|
||||
None,
|
||||
));
|
||||
session.invalidate();
|
||||
filter.set_session(session);
|
||||
|
||||
let mut packet = ZCPacket::new_with_payload(b"relay payload");
|
||||
packet.fill_peer_manager_hdr(my_peer_id, dst_peer_id, PacketType::Data as u8);
|
||||
packet
|
||||
.mut_peer_manager_header()
|
||||
.unwrap()
|
||||
.set_encrypted(true);
|
||||
let original_len = packet.buf_len();
|
||||
|
||||
let packet = filter
|
||||
.before_send(packet)
|
||||
.expect("relay packet should bypass next-hop session");
|
||||
|
||||
let hdr = packet.peer_manager_header().unwrap();
|
||||
assert_eq!(hdr.from_peer_id.get(), my_peer_id);
|
||||
assert_eq!(hdr.to_peer_id.get(), dst_peer_id);
|
||||
assert!(hdr.is_encrypted());
|
||||
assert_eq!(packet.buf_len(), original_len);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_conn_handshake_same_id() {
|
||||
let ps = Arc::new(PeerSessionStore::new());
|
||||
|
||||
@@ -6,7 +6,7 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use rand::{Rng, thread_rng};
|
||||
use tokio::{
|
||||
sync::broadcast,
|
||||
|
||||
@@ -2,7 +2,7 @@ use anyhow::Context;
|
||||
use async_trait::async_trait;
|
||||
use cidr::{Ipv4Cidr, Ipv6Cidr};
|
||||
use dashmap::DashMap;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use std::collections::BTreeSet;
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
@@ -11,6 +11,9 @@ use std::{
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
use hotpath::wrap::tokio::sync::{Mutex, RwLock};
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::{
|
||||
sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
|
||||
@@ -276,8 +279,8 @@ impl PeerManager {
|
||||
let rpc_tspt = Arc::new(RpcTransport {
|
||||
my_peer_id,
|
||||
peers: Arc::downgrade(&peers),
|
||||
foreign_peers: Mutex::new(None),
|
||||
packet_recv: Mutex::new(peer_rpc_tspt_recv),
|
||||
foreign_peers: hotpath::mutex!(tokio::sync::Mutex::new(None)),
|
||||
packet_recv: hotpath::mutex!(tokio::sync::Mutex::new(peer_rpc_tspt_recv)),
|
||||
peer_rpc_tspt_sender,
|
||||
encryptor: encryptor.clone(),
|
||||
is_secure_mode_enabled,
|
||||
@@ -409,17 +412,21 @@ impl PeerManager {
|
||||
global_ctx,
|
||||
nic_channel,
|
||||
|
||||
tasks: Mutex::new(JoinSet::new()),
|
||||
tasks: hotpath::mutex!(tokio::sync::Mutex::new(JoinSet::new())),
|
||||
|
||||
packet_recv: Arc::new(Mutex::new(Some(packet_recv))),
|
||||
packet_recv: Arc::new(hotpath::mutex!(tokio::sync::Mutex::new(Some(packet_recv)))),
|
||||
|
||||
peers,
|
||||
|
||||
peer_rpc_mgr,
|
||||
peer_rpc_tspt: rpc_tspt,
|
||||
|
||||
peer_packet_process_pipeline: Arc::new(RwLock::new(Vec::new())),
|
||||
nic_packet_process_pipeline: Arc::new(RwLock::new(Vec::new())),
|
||||
peer_packet_process_pipeline: Arc::new(hotpath::rw_lock!(tokio::sync::RwLock::new(
|
||||
Vec::new()
|
||||
))),
|
||||
nic_packet_process_pipeline: Arc::new(hotpath::rw_lock!(tokio::sync::RwLock::new(
|
||||
Vec::new()
|
||||
))),
|
||||
|
||||
route_algo_inst,
|
||||
|
||||
@@ -430,7 +437,7 @@ impl PeerManager {
|
||||
encryptor,
|
||||
data_compress_algo,
|
||||
|
||||
exit_nodes: RwLock::new(exit_nodes),
|
||||
exit_nodes: hotpath::rw_lock!(tokio::sync::RwLock::new(exit_nodes)),
|
||||
|
||||
reserved_my_peer_id_map: DashMap::new(),
|
||||
recent_have_traffic: Arc::new(DashMap::new()),
|
||||
@@ -1008,7 +1015,17 @@ impl PeerManager {
|
||||
|
||||
self.tasks.lock().await.spawn(async move {
|
||||
tracing::trace!("start_peer_recv");
|
||||
while let Ok(ret) = recv_packet_from_chan(&mut recv).await {
|
||||
loop {
|
||||
let ret = match recv.try_recv() {
|
||||
Ok(pkt) => pkt,
|
||||
Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
|
||||
match recv.recv().await {
|
||||
Some(pkt) => pkt,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => break,
|
||||
};
|
||||
let disable_relay_data = global_ctx.flags_arc().disable_relay_data;
|
||||
let Err(mut ret) = Self::try_handle_foreign_network_packet(
|
||||
ret,
|
||||
@@ -1146,9 +1163,11 @@ impl PeerManager {
|
||||
|
||||
self_rx_bytes.add(buf_len as u64);
|
||||
self_rx_packets.inc();
|
||||
traffic_metrics
|
||||
.record_rx(from_peer_id, packet_type, buf_len as u64)
|
||||
.await;
|
||||
if !traffic_metrics.record_rx_fast(from_peer_id, packet_type, buf_len as u64) {
|
||||
traffic_metrics
|
||||
.record_rx(from_peer_id, packet_type, buf_len as u64)
|
||||
.await;
|
||||
}
|
||||
compress_rx_bytes_before.add(buf_len as u64);
|
||||
|
||||
let compressor = DefaultCompressor {};
|
||||
@@ -1437,6 +1456,7 @@ impl PeerManager {
|
||||
self.get_route().get_foreign_network_summary().await
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerManager"))]
|
||||
async fn run_nic_packet_process_pipeline(&self, data: &mut ZCPacket) -> bool {
|
||||
// Enforce ACL for outbound (NIC-originated) packets. If ACL denies, stop processing.
|
||||
if !self.global_ctx.get_acl_filter().process_packet_with_acl(
|
||||
@@ -1522,6 +1542,7 @@ impl PeerManager {
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerManager"))]
|
||||
async fn send_msg_internal(
|
||||
peers: &Arc<PeerMap>,
|
||||
foreign_network_client: &Arc<ForeignNetworkClient>,
|
||||
@@ -1547,8 +1568,8 @@ impl PeerManager {
|
||||
&& (peers.has_peer(gateway) || foreign_network_client.has_next_hop(gateway))
|
||||
{
|
||||
relay_peer_map.send_msg(msg, dst_peer_id, policy).await
|
||||
} else if peers.has_peer(dst_peer_id) {
|
||||
peers.send_msg_directly(msg, dst_peer_id).await
|
||||
} else if let Some(peer) = peers.get_peer_by_id(dst_peer_id) {
|
||||
peer.send_msg(msg).await
|
||||
} else if foreign_network_client.has_next_hop(dst_peer_id) {
|
||||
foreign_network_client.send_msg(msg, dst_peer_id).await
|
||||
} else if let Some(gateway) = peers.get_gateway_peer_id(dst_peer_id, policy.clone()).await {
|
||||
@@ -1573,7 +1594,9 @@ impl PeerManager {
|
||||
if send_result.is_ok()
|
||||
&& let Some(metrics) = direct_tx_metrics
|
||||
{
|
||||
metrics.record_tx(dst_peer_id, packet_type, msg_len).await;
|
||||
if !metrics.record_tx_fast(dst_peer_id, packet_type, msg_len) {
|
||||
metrics.record_tx(dst_peer_id, packet_type, msg_len).await;
|
||||
}
|
||||
}
|
||||
|
||||
send_result
|
||||
@@ -1620,6 +1643,7 @@ impl PeerManager {
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerManager"))]
|
||||
pub async fn get_msg_dst_peer_ipv4(&self, ipv4_addr: &Ipv4Addr) -> (Vec<PeerId>, bool) {
|
||||
let mut is_exit_node = false;
|
||||
let mut dst_peers = vec![];
|
||||
@@ -1688,6 +1712,7 @@ impl PeerManager {
|
||||
(dst_peers, is_exit_node)
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerManager"))]
|
||||
pub async fn try_compress_and_encrypt(
|
||||
compress_algo: CompressorAlgo,
|
||||
encryptor: &Arc<dyn Encryptor + 'static>,
|
||||
@@ -1705,6 +1730,7 @@ impl PeerManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerManager"))]
|
||||
pub async fn send_msg_by_ip(
|
||||
&self,
|
||||
mut msg: ZCPacket,
|
||||
@@ -1968,6 +1994,18 @@ impl PeerManager {
|
||||
self.peers.clone()
|
||||
}
|
||||
|
||||
pub fn set_peer_conn_batch_threshold(&self, n: u32) {
|
||||
let peers = self.peers.clone();
|
||||
tokio::spawn(async move {
|
||||
let peer_ids = peers.list_peers();
|
||||
for peer_id in peer_ids {
|
||||
if let Some(peer) = peers.get_peer_by_id(peer_id) {
|
||||
peer.set_batch_threshold(n);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn get_relay_peer_map(&self) -> Arc<RelayPeerMap> {
|
||||
self.relay_peer_map.clone()
|
||||
}
|
||||
@@ -2198,7 +2236,7 @@ mod tests {
|
||||
use base64::Engine;
|
||||
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
|
||||
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
|
||||
use crate::{
|
||||
common::{
|
||||
|
||||
@@ -38,6 +38,7 @@ pub struct PeerMap {
|
||||
alive_client_urls: Arc<Mutex<multimap::MultiMap<url::Url, PeerConnId>>>,
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure_all)]
|
||||
impl PeerMap {
|
||||
pub fn new(packet_send: PacketRecvChan, global_ctx: ArcGlobalCtx, my_peer_id: PeerId) -> Self {
|
||||
PeerMap {
|
||||
|
||||
@@ -13,6 +13,7 @@ use arc_swap::ArcSwap;
|
||||
use cidr::{IpCidr, Ipv4Cidr, Ipv6Cidr, Ipv6Inet};
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use ordered_hash_map::OrderedHashMap;
|
||||
use parking_lot::{RwLock, lock_api::RwLockUpgradableReadGuard};
|
||||
use petgraph::{
|
||||
@@ -24,7 +25,6 @@ use petgraph::{
|
||||
use prefix_trie::PrefixMap;
|
||||
use prost::Message;
|
||||
use prost_reflect::{DynamicMessage, ReflectMessage};
|
||||
use quanta::Instant;
|
||||
use tokio::{
|
||||
select,
|
||||
sync::Mutex,
|
||||
@@ -1394,6 +1394,7 @@ impl RouteTable {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "RouteTable"))]
|
||||
fn get_next_hop(&self, dst_peer_id: PeerId) -> Option<NextHopInfo> {
|
||||
if self.suppressed_peer_ids.contains_key(&dst_peer_id) {
|
||||
return None;
|
||||
@@ -1401,6 +1402,7 @@ impl RouteTable {
|
||||
self.get_topology_next_hop(dst_peer_id)
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "RouteTable"))]
|
||||
fn get_topology_next_hop(&self, dst_peer_id: PeerId) -> Option<NextHopInfo> {
|
||||
let cur_version = self.next_hop_map_version.get();
|
||||
self.next_hop_map.get(&dst_peer_id).and_then(|x| {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::net::{IpAddr, Ipv6Addr, SocketAddr};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use crate::{
|
||||
common::{global_ctx::ArcGlobalCtx, network::IPCollector},
|
||||
common::global_ctx::ArcGlobalCtx,
|
||||
proto::{
|
||||
common::Void,
|
||||
peer_rpc::{
|
||||
@@ -12,8 +12,6 @@ use crate::{
|
||||
tunnel::udp,
|
||||
};
|
||||
|
||||
const MAX_UDP_HOLE_PUNCH_CONNECTOR_ADDRS: usize = 16;
|
||||
|
||||
fn remove_easytier_managed_ipv6s(ret: &mut GetIpListResponse, global_ctx: &ArcGlobalCtx) {
|
||||
ret.interface_ipv6s.retain(|ip| {
|
||||
let ip = std::net::Ipv6Addr::from(*ip);
|
||||
@@ -30,86 +28,6 @@ fn remove_easytier_managed_ipv6s(ret: &mut GetIpListResponse, global_ctx: &ArcGl
|
||||
}
|
||||
}
|
||||
|
||||
fn is_usable_preferred_src_ipv6(ip: &Ipv6Addr, global_ctx: &ArcGlobalCtx) -> bool {
|
||||
!global_ctx.is_ip_easytier_managed_ipv6(ip)
|
||||
&& !ip.is_loopback()
|
||||
&& !ip.is_unspecified()
|
||||
&& !ip.is_unique_local()
|
||||
&& !ip.is_unicast_link_local()
|
||||
&& !ip.is_multicast()
|
||||
}
|
||||
|
||||
async fn local_preferred_src_ipv6(
|
||||
global_ctx: &ArcGlobalCtx,
|
||||
preferred_src_ipv6: Option<crate::proto::common::Ipv6Addr>,
|
||||
) -> Option<udp::PreferredIpv6Source> {
|
||||
let preferred_src_ipv6 = preferred_src_ipv6.map(Ipv6Addr::from)?;
|
||||
if !is_usable_preferred_src_ipv6(&preferred_src_ipv6, global_ctx) {
|
||||
tracing::debug!(
|
||||
?preferred_src_ipv6,
|
||||
"ignore unusable preferred IPv6 source for udp hole punch"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
let ifaces = IPCollector::collect_interfaces(global_ctx.net_ns.clone(), false).await;
|
||||
for iface in ifaces {
|
||||
let is_local = iface.ips.iter().any(|ip| match ip.ip() {
|
||||
IpAddr::V6(v6) => v6 == preferred_src_ipv6,
|
||||
IpAddr::V4(_) => false,
|
||||
});
|
||||
if is_local {
|
||||
tracing::debug!(
|
||||
?preferred_src_ipv6,
|
||||
ifindex = iface.index,
|
||||
"use preferred IPv6 source for udp hole punch"
|
||||
);
|
||||
return Some(udp::PreferredIpv6Source {
|
||||
ip: preferred_src_ipv6,
|
||||
ifindex: iface.index,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
?preferred_src_ipv6,
|
||||
"ignore non-local preferred IPv6 source for udp hole punch"
|
||||
);
|
||||
None
|
||||
}
|
||||
|
||||
fn connector_addrs_from_request(
|
||||
req: SendUdpHolePunchPacketRequest,
|
||||
) -> rpc_types::error::Result<(u16, Vec<SocketAddr>, Option<crate::proto::common::Ipv6Addr>)> {
|
||||
let listener_port = u16::try_from(req.listener_port)
|
||||
.map_err(|_| anyhow::anyhow!("listener_port is out of range: {}", req.listener_port))?;
|
||||
let mut connector_addrs = req
|
||||
.connector_addrs
|
||||
.into_iter()
|
||||
.map(SocketAddr::from)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if connector_addrs.is_empty() {
|
||||
connector_addrs.push(
|
||||
req.connector_addr
|
||||
.ok_or(anyhow::anyhow!("connector_addr is required"))?
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let mut deduped = Vec::with_capacity(connector_addrs.len());
|
||||
for addr in connector_addrs {
|
||||
if !deduped.contains(&addr) {
|
||||
deduped.push(addr);
|
||||
}
|
||||
if deduped.len() >= MAX_UDP_HOLE_PUNCH_CONNECTOR_ADDRS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((listener_port, deduped, req.preferred_src_ipv6))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DirectConnectorManagerRpcServer {
|
||||
// TODO: this only cache for one src peer, should make it global
|
||||
@@ -149,38 +67,23 @@ impl DirectConnectorRpc for DirectConnectorManagerRpcServer {
|
||||
_: BaseController,
|
||||
req: SendUdpHolePunchPacketRequest,
|
||||
) -> rpc_types::error::Result<Void> {
|
||||
let (listener_port, connector_addrs, preferred_src_ipv6) =
|
||||
connector_addrs_from_request(req)?;
|
||||
let preferred_src_ipv6 =
|
||||
local_preferred_src_ipv6(&self.global_ctx, preferred_src_ipv6).await;
|
||||
let listener_port = req.listener_port as u16;
|
||||
let connector_addr: SocketAddr = req
|
||||
.connector_addr
|
||||
.ok_or(anyhow::anyhow!("connector_addr is required"))?
|
||||
.into();
|
||||
|
||||
tracing::info!(
|
||||
?connector_addrs,
|
||||
?preferred_src_ipv6,
|
||||
listener_port,
|
||||
"Sending udp hole punch packet"
|
||||
"Sending udp hole punch packet to {} from listener port {}",
|
||||
connector_addr,
|
||||
listener_port
|
||||
);
|
||||
|
||||
// send 3 packets to the connector
|
||||
for _ in 0..3 {
|
||||
for connector_addr in &connector_addrs {
|
||||
let ret = match connector_addr {
|
||||
SocketAddr::V4(addr) => {
|
||||
udp::send_v4_hole_punch_packet(listener_port, *addr).await
|
||||
}
|
||||
SocketAddr::V6(addr) => {
|
||||
udp::send_v6_hole_punch_packet(listener_port, *addr, preferred_src_ipv6)
|
||||
.await
|
||||
}
|
||||
};
|
||||
if let Err(e) = ret {
|
||||
tracing::debug!(
|
||||
?e,
|
||||
?connector_addr,
|
||||
listener_port,
|
||||
"send udp hole punch packet failed"
|
||||
);
|
||||
}
|
||||
match connector_addr {
|
||||
SocketAddr::V4(addr) => udp::send_v4_hole_punch_packet(listener_port, addr).await?,
|
||||
SocketAddr::V6(addr) => udp::send_v6_hole_punch_packet(listener_port, addr).await?,
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
}
|
||||
@@ -196,12 +99,11 @@ impl DirectConnectorManagerRpcServer {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{collections::BTreeSet, net::SocketAddr};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::{
|
||||
common::global_ctx::tests::get_mock_global_ctx,
|
||||
peers::peer_rpc_service::{connector_addrs_from_request, remove_easytier_managed_ipv6s},
|
||||
proto::peer_rpc::{GetIpListResponse, SendUdpHolePunchPacketRequest},
|
||||
peers::peer_rpc_service::remove_easytier_managed_ipv6s, proto::peer_rpc::GetIpListResponse,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
@@ -231,58 +133,4 @@ mod tests {
|
||||
assert_eq!(ip_list.public_ipv6, None);
|
||||
assert_eq!(ip_list.interface_ipv6s, vec![physical_ipv6.into()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hole_punch_request_prefers_batch_connector_addrs() {
|
||||
let old_addr: SocketAddr = "[2001:db8::1]:10001".parse().unwrap();
|
||||
let first_batch_addr: SocketAddr = "[2001:db8::2]:10002".parse().unwrap();
|
||||
let second_batch_addr: SocketAddr = "[2001:db8::3]:10003".parse().unwrap();
|
||||
let preferred_src_ipv6: std::net::Ipv6Addr = "2001:db8::4".parse().unwrap();
|
||||
|
||||
let (listener_port, connector_addrs, preferred_src) =
|
||||
connector_addrs_from_request(SendUdpHolePunchPacketRequest {
|
||||
connector_addr: Some(old_addr.into()),
|
||||
listener_port: 11010,
|
||||
preferred_src_ipv6: Some(preferred_src_ipv6.into()),
|
||||
connector_addrs: vec![
|
||||
first_batch_addr.into(),
|
||||
first_batch_addr.into(),
|
||||
second_batch_addr.into(),
|
||||
],
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(listener_port, 11010);
|
||||
assert_eq!(connector_addrs, vec![first_batch_addr, second_batch_addr]);
|
||||
assert_eq!(preferred_src, Some(preferred_src_ipv6.into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hole_punch_request_falls_back_to_legacy_connector_addr() {
|
||||
let old_addr: SocketAddr = "[2001:db8::1]:10001".parse().unwrap();
|
||||
|
||||
let (_, connector_addrs, _) = connector_addrs_from_request(SendUdpHolePunchPacketRequest {
|
||||
connector_addr: Some(old_addr.into()),
|
||||
listener_port: 11010,
|
||||
preferred_src_ipv6: None,
|
||||
connector_addrs: vec![],
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(connector_addrs, vec![old_addr]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hole_punch_request_rejects_out_of_range_listener_port() {
|
||||
let old_addr: SocketAddr = "[2001:db8::1]:10001".parse().unwrap();
|
||||
|
||||
let ret = connector_addrs_from_request(SendUdpHolePunchPacketRequest {
|
||||
connector_addr: Some(old_addr.into()),
|
||||
listener_port: u16::MAX as u32 + 1,
|
||||
preferred_src_ipv6: None,
|
||||
connector_addrs: vec![],
|
||||
});
|
||||
|
||||
assert!(ret.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,9 @@ use std::sync::{
|
||||
Arc, RwLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use crossbeam::atomic::AtomicCell;
|
||||
use dashmap::DashMap;
|
||||
use quanta::Instant;
|
||||
|
||||
use super::secure_datagram::{SecureDatagramDirection, SecureDatagramSession};
|
||||
use crate::{
|
||||
@@ -15,8 +12,6 @@ use crate::{
|
||||
tunnel::packet_def::ZCPacket,
|
||||
};
|
||||
|
||||
const SESSION_IDLE_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
pub struct UpsertResponderSessionReturn {
|
||||
pub session: Arc<PeerSession>,
|
||||
pub action: PeerSessionAction,
|
||||
@@ -49,25 +44,7 @@ impl SessionKey {
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PeerSessionStore {
|
||||
sessions: Arc<DashMap<SessionKey, PeerSessionEntry>>,
|
||||
}
|
||||
|
||||
struct PeerSessionEntry {
|
||||
session: Arc<PeerSession>,
|
||||
last_used_at: AtomicCell<Instant>,
|
||||
}
|
||||
|
||||
impl PeerSessionEntry {
|
||||
fn new(session: Arc<PeerSession>) -> Self {
|
||||
Self {
|
||||
session,
|
||||
last_used_at: AtomicCell::new(Instant::now()),
|
||||
}
|
||||
}
|
||||
|
||||
fn touch(&self) {
|
||||
self.last_used_at.store(Instant::now());
|
||||
}
|
||||
sessions: Arc<DashMap<SessionKey, Arc<PeerSession>>>,
|
||||
}
|
||||
|
||||
impl Default for PeerSessionStore {
|
||||
@@ -84,11 +61,7 @@ impl PeerSessionStore {
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &SessionKey) -> Option<Arc<PeerSession>> {
|
||||
let session = {
|
||||
let entry = self.sessions.get(key)?;
|
||||
entry.touch();
|
||||
entry.session.clone()
|
||||
};
|
||||
let session = self.sessions.get(key)?.clone();
|
||||
if session.is_valid() {
|
||||
Some(session)
|
||||
} else {
|
||||
@@ -102,20 +75,12 @@ impl PeerSessionStore {
|
||||
}
|
||||
|
||||
pub fn insert_session(&self, key: SessionKey, session: Arc<PeerSession>) {
|
||||
self.sessions.insert(key, PeerSessionEntry::new(session));
|
||||
self.sessions.insert(key, session);
|
||||
}
|
||||
|
||||
pub fn evict_unused_sessions(&self) {
|
||||
self.evict_unused_sessions_idle(SESSION_IDLE_TIMEOUT);
|
||||
}
|
||||
|
||||
pub fn evict_unused_sessions_idle(&self, idle: Duration) {
|
||||
let now = Instant::now();
|
||||
self.sessions.retain(|_key, entry| {
|
||||
entry.session.is_valid()
|
||||
&& (Arc::strong_count(&entry.session) > 1
|
||||
|| now.saturating_duration_since(entry.last_used_at.load()) < idle)
|
||||
});
|
||||
self.sessions
|
||||
.retain(|_key, session| Arc::strong_count(session) > 1);
|
||||
shrink_dashmap(&self.sessions, None);
|
||||
}
|
||||
|
||||
@@ -128,14 +93,11 @@ impl PeerSessionStore {
|
||||
recv_algorithm: String,
|
||||
peer_static_pubkey: Option<[u8; 32]>,
|
||||
) -> Result<UpsertResponderSessionReturn, anyhow::Error> {
|
||||
tracing::event!(tracing::Level::INFO, ?key, "upsert_responder_session");
|
||||
tracing::event!(tracing::Level::INFO, "upsert_responder_session {:?}", key);
|
||||
let existing = self
|
||||
.sessions
|
||||
.get(key)
|
||||
.map(|v| {
|
||||
v.touch();
|
||||
v.session.clone()
|
||||
})
|
||||
.map(|v| v.clone())
|
||||
.filter(|s| s.is_valid());
|
||||
match existing {
|
||||
None => {
|
||||
@@ -151,8 +113,7 @@ impl PeerSessionStore {
|
||||
recv_algorithm,
|
||||
peer_static_pubkey,
|
||||
));
|
||||
self.sessions
|
||||
.insert(key.clone(), PeerSessionEntry::new(session.clone()));
|
||||
self.sessions.insert(key.clone(), session.clone());
|
||||
Ok(UpsertResponderSessionReturn {
|
||||
session,
|
||||
action: PeerSessionAction::Create,
|
||||
@@ -217,14 +178,16 @@ impl PeerSessionStore {
|
||||
PeerSessionAction::Sync | PeerSessionAction::Create => {
|
||||
let root_key = root_key_32.ok_or_else(|| anyhow!("missing root_key"))?;
|
||||
if let Some(existing) = self.sessions.get(key)
|
||||
&& !existing.session.is_valid()
|
||||
&& !existing.is_valid()
|
||||
{
|
||||
drop(existing);
|
||||
self.sessions.remove(key);
|
||||
}
|
||||
let session = {
|
||||
let entry = self.sessions.entry(key.clone()).or_insert_with(|| {
|
||||
PeerSessionEntry::new(Arc::new(PeerSession::new(
|
||||
let session = self
|
||||
.sessions
|
||||
.entry(key.clone())
|
||||
.or_insert_with(|| {
|
||||
Arc::new(PeerSession::new(
|
||||
key.peer_id,
|
||||
root_key,
|
||||
b_session_generation,
|
||||
@@ -232,11 +195,9 @@ impl PeerSessionStore {
|
||||
send_algorithm.clone(),
|
||||
recv_algorithm.clone(),
|
||||
peer_static_pubkey,
|
||||
)))
|
||||
});
|
||||
entry.touch();
|
||||
entry.session.clone()
|
||||
};
|
||||
))
|
||||
})
|
||||
.clone();
|
||||
session.check_encrypt_algo_same(&send_algorithm, &recv_algorithm)?;
|
||||
session.check_or_set_peer_static_pubkey(peer_static_pubkey)?;
|
||||
session.sync_root_key(
|
||||
@@ -376,6 +337,7 @@ impl PeerSession {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerSession"))]
|
||||
pub fn encrypt_payload(
|
||||
&self,
|
||||
sender_peer_id: PeerId,
|
||||
@@ -389,6 +351,7 @@ impl PeerSession {
|
||||
.encrypt_payload(Self::dir_for_sender(sender_peer_id, receiver_peer_id), pkt)
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "PeerSession"))]
|
||||
pub fn decrypt_payload(
|
||||
&self,
|
||||
sender_peer_id: PeerId,
|
||||
@@ -458,77 +421,4 @@ mod tests {
|
||||
SecureDatagramSession::SYNC_RX_GRACE_AFTER_MS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_session_store_keeps_recent_session_without_external_refs() {
|
||||
let store = PeerSessionStore::new();
|
||||
let key = SessionKey::new("net".to_string(), 20);
|
||||
let session = Arc::new(PeerSession::new(
|
||||
20,
|
||||
PeerSession::new_root_key(),
|
||||
1,
|
||||
0,
|
||||
"aes-gcm".to_string(),
|
||||
"aes-gcm".to_string(),
|
||||
None,
|
||||
));
|
||||
store.insert_session(key.clone(), session);
|
||||
|
||||
assert!(store.get(&key).is_some());
|
||||
store.evict_unused_sessions();
|
||||
|
||||
assert!(
|
||||
store.get(&key).is_some(),
|
||||
"recent relay sessions should survive the periodic GC"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_session_store_evicts_idle_session_without_external_refs() {
|
||||
let store = PeerSessionStore::new();
|
||||
let key = SessionKey::new("net".to_string(), 20);
|
||||
let session = Arc::new(PeerSession::new(
|
||||
20,
|
||||
PeerSession::new_root_key(),
|
||||
1,
|
||||
0,
|
||||
"aes-gcm".to_string(),
|
||||
"aes-gcm".to_string(),
|
||||
None,
|
||||
));
|
||||
store.insert_session(key.clone(), session);
|
||||
|
||||
store.evict_unused_sessions_idle(Duration::from_millis(0));
|
||||
|
||||
assert!(
|
||||
store.get(&key).is_none(),
|
||||
"idle sessions without external users should still be collected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_session_store_evicts_invalid_recent_session() {
|
||||
let store = PeerSessionStore::new();
|
||||
let key = SessionKey::new("net".to_string(), 20);
|
||||
let session = Arc::new(PeerSession::new(
|
||||
20,
|
||||
PeerSession::new_root_key(),
|
||||
1,
|
||||
0,
|
||||
"aes-gcm".to_string(),
|
||||
"aes-gcm".to_string(),
|
||||
None,
|
||||
));
|
||||
store.insert_session(key.clone(), session);
|
||||
|
||||
let session = store.get(&key).unwrap();
|
||||
session.invalidate();
|
||||
drop(session);
|
||||
store.evict_unused_sessions();
|
||||
|
||||
assert!(
|
||||
!store.sessions.contains_key(&key),
|
||||
"invalid sessions should not be kept by recent activity"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use prost::Message;
|
||||
use quanta::Instant;
|
||||
use snow::params::NoiseParams;
|
||||
use tokio::sync::{Mutex, OwnedMutexGuard, oneshot};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use cidr::Ipv6Inet;
|
||||
use cidr::{Ipv4Cidr, Ipv6Cidr};
|
||||
use dashmap::DashMap;
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
net::{Ipv4Addr, Ipv6Addr},
|
||||
|
||||
@@ -701,6 +701,10 @@ impl SecureDatagramSession {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
feature = "hotpath",
|
||||
hotpath::measure(impl_type = "SecureDatagramSession")
|
||||
)]
|
||||
pub fn encrypt_payload(
|
||||
&self,
|
||||
dir: SecureDatagramDirection,
|
||||
@@ -719,6 +723,10 @@ impl SecureDatagramSession {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
feature = "hotpath",
|
||||
hotpath::measure(impl_type = "SecureDatagramSession")
|
||||
)]
|
||||
pub fn decrypt_payload(
|
||||
&self,
|
||||
dir: SecureDatagramDirection,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::{future::Future, sync::Arc};
|
||||
use std::{future::Future, sync::atomic::{AtomicU64, Ordering}, sync::Arc};
|
||||
|
||||
use dashmap::DashMap;
|
||||
use futures::future::BoxFuture;
|
||||
@@ -18,16 +18,54 @@ pub(crate) enum InstanceLabelKind {
|
||||
From,
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
const TRAFFIC_BATCH_SIZE: u64 = 128;
|
||||
#[cfg(test)]
|
||||
const TRAFFIC_BATCH_SIZE: u64 = 1;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TrafficCounters {
|
||||
bytes: CounterHandle,
|
||||
packets: CounterHandle,
|
||||
batch: Arc<TrafficBatch>,
|
||||
}
|
||||
|
||||
struct TrafficBatch {
|
||||
bytes: AtomicU64,
|
||||
packets: AtomicU64,
|
||||
}
|
||||
|
||||
impl TrafficCounters {
|
||||
fn new(bytes: CounterHandle, packets: CounterHandle) -> Self {
|
||||
Self {
|
||||
bytes,
|
||||
packets,
|
||||
batch: Arc::new(TrafficBatch {
|
||||
bytes: AtomicU64::new(0),
|
||||
packets: AtomicU64::new(0),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn add_sample(&self, bytes: u64) {
|
||||
self.bytes.add(bytes);
|
||||
self.packets.inc();
|
||||
let prev = self.batch.packets.fetch_add(1, Ordering::Relaxed);
|
||||
self.batch.bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
if (prev + 1) % TRAFFIC_BATCH_SIZE == 0 {
|
||||
let b = self.batch.bytes.swap(0, Ordering::Relaxed);
|
||||
self.bytes.add(b);
|
||||
self.packets.add(TRAFFIC_BATCH_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&self) {
|
||||
let b = self.batch.bytes.swap(0, Ordering::Relaxed);
|
||||
let p = self.batch.packets.swap(0, Ordering::Relaxed);
|
||||
if b > 0 {
|
||||
self.bytes.add(b);
|
||||
}
|
||||
if p > 0 {
|
||||
self.packets.add(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,14 +98,14 @@ impl AggregateTrafficMetrics {
|
||||
let label_set =
|
||||
LabelSet::new().with_label_type(LabelType::NetworkName(network_name.clone()));
|
||||
Self {
|
||||
tx: TrafficCounters {
|
||||
bytes: stats_mgr.get_counter(tx_bytes_metric, label_set.clone()),
|
||||
packets: stats_mgr.get_counter(tx_packets_metric, label_set.clone()),
|
||||
},
|
||||
rx: TrafficCounters {
|
||||
bytes: stats_mgr.get_counter(rx_bytes_metric, label_set.clone()),
|
||||
packets: stats_mgr.get_counter(rx_packets_metric, label_set),
|
||||
},
|
||||
tx: TrafficCounters::new(
|
||||
stats_mgr.get_counter(tx_bytes_metric, label_set.clone()),
|
||||
stats_mgr.get_counter(tx_packets_metric, label_set.clone()),
|
||||
),
|
||||
rx: TrafficCounters::new(
|
||||
stats_mgr.get_counter(rx_bytes_metric, label_set.clone()),
|
||||
stats_mgr.get_counter(rx_packets_metric, label_set),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,10 +160,10 @@ impl LogicalTrafficMetrics {
|
||||
let label_set =
|
||||
LabelSet::new().with_label_type(LabelType::NetworkName(network_name.clone()));
|
||||
Self {
|
||||
total: TrafficCounters {
|
||||
bytes: stats_mgr.get_counter(total_bytes_metric, label_set.clone()),
|
||||
packets: stats_mgr.get_counter(total_packets_metric, label_set),
|
||||
},
|
||||
total: TrafficCounters::new(
|
||||
stats_mgr.get_counter(total_bytes_metric, label_set.clone()),
|
||||
stats_mgr.get_counter(total_packets_metric, label_set),
|
||||
),
|
||||
stats_mgr,
|
||||
network_name,
|
||||
instance_bytes_metric,
|
||||
@@ -135,6 +173,22 @@ impl LogicalTrafficMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_fast(&self, peer_id: PeerId, bytes: u64) -> bool {
|
||||
self.total.add_sample(bytes);
|
||||
if let Some(entry) = self.per_peer.get(&peer_id)
|
||||
&& entry.value().is_resolved()
|
||||
{
|
||||
let counters = match entry.value() {
|
||||
CachedPeerTrafficCounters::Resolved(c)
|
||||
| CachedPeerTrafficCounters::Unknown(c) => c,
|
||||
};
|
||||
counters.add_sample(bytes);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn record_with_resolver<F, Fut>(
|
||||
&self,
|
||||
peer_id: PeerId,
|
||||
@@ -214,14 +268,12 @@ impl LogicalTrafficMetrics {
|
||||
let label_set = LabelSet::new()
|
||||
.with_label_type(LabelType::NetworkName(self.network_name.clone()))
|
||||
.with_label_type(instance_label);
|
||||
TrafficCounters {
|
||||
bytes: self
|
||||
.stats_mgr
|
||||
TrafficCounters::new(
|
||||
self.stats_mgr
|
||||
.get_counter(self.instance_bytes_metric, label_set.clone()),
|
||||
packets: self
|
||||
.stats_mgr
|
||||
self.stats_mgr
|
||||
.get_counter(self.instance_packets_metric, label_set),
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,6 +356,15 @@ impl TrafficMetricRecorder {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_tx_fast(&self, peer_id: PeerId, packet_type: u8, bytes: u64) -> bool {
|
||||
if peer_id == self.my_peer_id {
|
||||
return true;
|
||||
}
|
||||
self.tx_metrics
|
||||
.select(traffic_kind(packet_type))
|
||||
.record_fast(peer_id, bytes)
|
||||
}
|
||||
|
||||
pub(crate) async fn record_tx(&self, peer_id: PeerId, packet_type: u8, bytes: u64) {
|
||||
if peer_id == self.my_peer_id {
|
||||
return;
|
||||
@@ -314,6 +375,15 @@ impl TrafficMetricRecorder {
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) fn record_rx_fast(&self, peer_id: PeerId, packet_type: u8, bytes: u64) -> bool {
|
||||
if peer_id == self.my_peer_id {
|
||||
return true;
|
||||
}
|
||||
self.rx_metrics
|
||||
.select(traffic_kind(packet_type))
|
||||
.record_fast(peer_id, bytes)
|
||||
}
|
||||
|
||||
pub(crate) async fn record_rx(&self, peer_id: PeerId, packet_type: u8, bytes: u64) {
|
||||
if peer_id == self.my_peer_id {
|
||||
return;
|
||||
|
||||
@@ -102,12 +102,6 @@ message NetworkConfig {
|
||||
optional bool disable_relay_data = 65;
|
||||
optional bool enable_udp_broadcast_relay = 66;
|
||||
optional uint32 socket_mark = 67;
|
||||
repeated NetworkPeerConfig peers = 68;
|
||||
}
|
||||
|
||||
message NetworkPeerConfig {
|
||||
string uri = 1;
|
||||
optional string peer_public_key = 2;
|
||||
}
|
||||
|
||||
message PortForwardConfig {
|
||||
|
||||
@@ -189,8 +189,6 @@ message GetIpListResponse {
|
||||
message SendUdpHolePunchPacketRequest {
|
||||
common.SocketAddr connector_addr = 1;
|
||||
uint32 listener_port = 2;
|
||||
common.Ipv6Addr preferred_src_ipv6 = 3;
|
||||
repeated common.SocketAddr connector_addrs = 4;
|
||||
}
|
||||
|
||||
service DirectConnectorRpc {
|
||||
|
||||
@@ -5,8 +5,8 @@ use std::sync::{Arc, Mutex};
|
||||
use bytes::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use guarden::defer;
|
||||
use hotpath::instant::Instant;
|
||||
use prost::Message;
|
||||
use quanta::Instant;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use prost::{Message as _, length_delimiter_len};
|
||||
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
|
||||
use crate::{
|
||||
common::{PeerId, compressor::DefaultCompressor},
|
||||
|
||||
@@ -5,8 +5,8 @@ use std::{
|
||||
|
||||
use bytes::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use hotpath::instant::Instant;
|
||||
use prost::Message;
|
||||
use quanta::Instant;
|
||||
use tokio::{task::JoinSet, time::timeout};
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
|
||||
@@ -2121,124 +2121,6 @@ pub async fn port_forward_test(
|
||||
drop_insts(_insts).await;
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case(false, false)]
|
||||
#[case(true, false)]
|
||||
#[case(true, true)]
|
||||
#[serial_test::serial]
|
||||
#[tokio::test]
|
||||
pub async fn port_forward_with_inbound_default_drop_acl_test(
|
||||
#[case] dhcp: bool,
|
||||
#[case] enable_quic_proxy: bool,
|
||||
) {
|
||||
use crate::proto::acl::*;
|
||||
|
||||
let acl = Acl {
|
||||
acl_v1: Some(AclV1 {
|
||||
chains: vec![Chain {
|
||||
name: "drop_unsolicited_inbound".to_string(),
|
||||
chain_type: ChainType::Inbound as i32,
|
||||
enabled: true,
|
||||
default_action: Action::Drop as i32,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
}),
|
||||
};
|
||||
|
||||
let insts = init_three_node_ex(
|
||||
"udp",
|
||||
|cfg| {
|
||||
if cfg.get_inst_name() == "inst1" {
|
||||
if dhcp {
|
||||
cfg.set_ipv4(None);
|
||||
cfg.set_dhcp(true);
|
||||
}
|
||||
cfg.set_acl(Some(acl.clone()));
|
||||
cfg.set_port_forwards(vec![
|
||||
PortForwardConfig {
|
||||
bind_addr: "0.0.0.0:23456".parse().unwrap(),
|
||||
dst_addr: "10.144.144.3:23456".parse().unwrap(),
|
||||
proto: "tcp".to_string(),
|
||||
},
|
||||
PortForwardConfig {
|
||||
bind_addr: "0.0.0.0:23457".parse().unwrap(),
|
||||
dst_addr: "10.1.2.4:23457".parse().unwrap(),
|
||||
proto: "tcp".to_string(),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut flags = cfg.get_flags();
|
||||
flags.no_tun = true;
|
||||
flags.enable_kcp_proxy = false;
|
||||
flags.enable_quic_proxy = enable_quic_proxy;
|
||||
cfg.set_flags(flags);
|
||||
} else if cfg.get_inst_name() == "inst3" {
|
||||
cfg.add_proxy_cidr("10.1.2.0/24".parse().unwrap(), None)
|
||||
.unwrap();
|
||||
|
||||
let mut flags = cfg.get_flags();
|
||||
flags.disable_kcp_input = true;
|
||||
flags.disable_quic_input = !enable_quic_proxy;
|
||||
cfg.set_flags(flags);
|
||||
} else if cfg.get_inst_name() == "inst2" {
|
||||
let mut flags = cfg.get_flags();
|
||||
flags.disable_relay_kcp = true;
|
||||
cfg.set_flags(flags);
|
||||
}
|
||||
|
||||
cfg
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
|
||||
if dhcp {
|
||||
wait_for_condition(
|
||||
|| async { insts[0].get_global_ctx().get_ipv4().is_some() },
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
for (bind_port, server_ns) in [(23456, "net_c"), (23457, "net_d")] {
|
||||
let tcp_listener =
|
||||
TcpTunnelListener::new(format!("tcp://0.0.0.0:{bind_port}").parse().unwrap());
|
||||
let tcp_connector =
|
||||
TcpTunnelConnector::new(format!("tcp://127.0.0.1:{bind_port}").parse().unwrap());
|
||||
|
||||
let mut buf = vec![0; 64];
|
||||
rand::thread_rng().fill(&mut buf[..]);
|
||||
|
||||
let result = _tunnel_pingpong_netns_with_timeout(
|
||||
tcp_listener,
|
||||
tcp_connector,
|
||||
NetNS::new(Some(server_ns.into())),
|
||||
NetNS::new(Some("net_a".into())),
|
||||
buf,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
|
||||
let stats = insts[0].get_global_ctx().get_acl_filter().get_stats();
|
||||
println!(
|
||||
"port forward source bind_port={} dhcp={} enable_quic_proxy={} ACL stats: {}",
|
||||
bind_port, dhcp, enable_quic_proxy, stats
|
||||
);
|
||||
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"port-forward TCP should complete through outbound ACL state, bind_port={}, dhcp={}, enable_quic_proxy={}; stats: {}",
|
||||
bind_port,
|
||||
dhcp,
|
||||
enable_quic_proxy,
|
||||
stats,
|
||||
);
|
||||
}
|
||||
|
||||
drop_insts(insts).await;
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[serial_test::serial]
|
||||
#[tokio::test]
|
||||
@@ -3771,25 +3653,6 @@ pub async fn config_patch_test() {
|
||||
true
|
||||
},
|
||||
);
|
||||
let patch = InstanceConfigPatch {
|
||||
proxy_networks: vec![ProxyNetworkPatch {
|
||||
action: ConfigPatchAction::Clear as i32,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
insts[1]
|
||||
.get_config_patcher()
|
||||
.apply_patch(patch)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
insts[1]
|
||||
.get_global_ctx()
|
||||
.config
|
||||
.get_proxy_cidrs()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// 测试1.1:修改公网 IPv6 provider 相关配置
|
||||
let public_prefix = "2001:db8:100::/64";
|
||||
@@ -4340,7 +4203,7 @@ pub async fn relay_peer_session_cleanup() {
|
||||
insts[0]
|
||||
.get_peer_manager()
|
||||
.get_peer_session_store()
|
||||
.evict_unused_sessions_idle(Duration::from_millis(0));
|
||||
.evict_unused_sessions();
|
||||
|
||||
wait_for_condition(
|
||||
|| async { !relay_map_1.has_session(inst3_peer_id) },
|
||||
|
||||
@@ -159,7 +159,7 @@ impl Socket {
|
||||
ack: Option<u32>,
|
||||
state: State,
|
||||
) -> (Socket, flume::Sender<Bytes>) {
|
||||
let (incoming_tx, incoming_rx) = flume::bounded(MPMC_BUFFER_LEN);
|
||||
let (incoming_tx, incoming_rx) = hotpath::channel!(flume::bounded(MPMC_BUFFER_LEN));
|
||||
|
||||
(
|
||||
Socket {
|
||||
|
||||
@@ -25,7 +25,6 @@ pub mod ring;
|
||||
pub mod stats;
|
||||
pub mod tcp;
|
||||
pub mod udp;
|
||||
pub(crate) mod udp_src;
|
||||
|
||||
#[cfg(feature = "faketcp")]
|
||||
pub mod fake_tcp;
|
||||
|
||||
+44
-12
@@ -4,7 +4,7 @@ use std::{
|
||||
cell::UnsafeCell,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
sync::atomic::{AtomicBool, AtomicU32, Ordering},
|
||||
task::Poll,
|
||||
time::Duration,
|
||||
};
|
||||
@@ -26,6 +26,8 @@ use futures::SinkExt;
|
||||
struct SpinSink {
|
||||
locked: AtomicBool,
|
||||
sink: UnsafeCell<Pin<Box<dyn ZCPacketSink>>>,
|
||||
pending_count: AtomicU32,
|
||||
batch_threshold: AtomicU32,
|
||||
}
|
||||
|
||||
// SAFETY: access is serialized by the spinlock.
|
||||
@@ -55,9 +57,15 @@ impl SpinSink {
|
||||
Self {
|
||||
locked: AtomicBool::new(false),
|
||||
sink: UnsafeCell::new(sink),
|
||||
pending_count: AtomicU32::new(0),
|
||||
batch_threshold: AtomicU32::new(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_batch_threshold(&self, n: u32) {
|
||||
self.batch_threshold.store(n, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn try_lock(&self) -> Option<SpinGuard<'_>> {
|
||||
if self
|
||||
.locked
|
||||
@@ -75,9 +83,11 @@ impl SpinSink {
|
||||
pub struct MpscTunnelSender {
|
||||
channel_tx: Option<Sender<ZCPacket>>,
|
||||
direct_sink: Option<Arc<SpinSink>>,
|
||||
direct_batch_flush: bool,
|
||||
}
|
||||
|
||||
impl MpscTunnelSender {
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "MpscTunnelSender"))]
|
||||
pub async fn send(&self, item: ZCPacket) -> Result<(), TunnelError> {
|
||||
if let Some(sink) = &self.direct_sink {
|
||||
// Sync fast path: no await needed, returns immediately
|
||||
@@ -87,13 +97,19 @@ impl MpscTunnelSender {
|
||||
match guard.as_mut().poll_ready(&mut cx) {
|
||||
Poll::Ready(Ok(())) => {
|
||||
guard.as_mut().start_send(item)?;
|
||||
// poll_flush may return Pending when the consumer task hasn't
|
||||
// drained the ring yet. The data is already in the ring buffer
|
||||
// and will be consumed — treat Pending as success.
|
||||
match guard.as_mut().poll_flush(&mut cx) {
|
||||
Poll::Ready(Err(e)) => return Err(e),
|
||||
_ => return Ok(()),
|
||||
let count = sink.pending_count.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
let threshold = sink.batch_threshold.load(Ordering::Relaxed);
|
||||
if count >= threshold {
|
||||
sink.pending_count.store(0, Ordering::Relaxed);
|
||||
// Batch flush: writev all accumulated BufList entries.
|
||||
// RingSink: no-op. FramedWriter: single writev syscall.
|
||||
match guard.as_mut().poll_flush(&mut cx) {
|
||||
Poll::Ready(Err(e)) => return Err(e),
|
||||
_ => return Ok(()),
|
||||
}
|
||||
}
|
||||
// Accumulate in BufList, no flush yet
|
||||
return Ok(());
|
||||
}
|
||||
Poll::Ready(Err(e)) => return Err(e),
|
||||
Poll::Pending => return Err(TunnelError::BufferFull),
|
||||
@@ -114,6 +130,12 @@ impl MpscTunnelSender {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn set_batch_threshold(&self, n: u32) {
|
||||
if let Some(sink) = &self.direct_sink {
|
||||
sink.set_batch_threshold(n);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn send_async(&self, item: ZCPacket) -> Result<(), TunnelError> {
|
||||
let tx = self.channel_tx.as_ref().ok_or(TunnelError::Shutdown)?;
|
||||
match tx.try_send(item) {
|
||||
@@ -130,6 +152,7 @@ impl MpscTunnelSender {
|
||||
pub struct MpscTunnel<T> {
|
||||
tx: Option<Sender<ZCPacket>>,
|
||||
direct_sink: Option<Arc<SpinSink>>,
|
||||
direct_batch_flush: bool,
|
||||
|
||||
tunnel: T,
|
||||
stream: Option<Pin<Box<dyn ZCPacketStream>>>,
|
||||
@@ -139,7 +162,7 @@ pub struct MpscTunnel<T> {
|
||||
|
||||
impl<T: Tunnel> MpscTunnel<T> {
|
||||
pub fn new(tunnel: T, send_timeout: Option<Duration>) -> Self {
|
||||
let (tx, mut rx) = channel(32);
|
||||
let (tx, mut rx) = hotpath::channel!(channel(1024));
|
||||
let (stream, mut sink) = tunnel.split();
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
@@ -157,6 +180,7 @@ impl<T: Tunnel> MpscTunnel<T> {
|
||||
Self {
|
||||
tx: Some(tx),
|
||||
direct_sink: None,
|
||||
direct_batch_flush: false,
|
||||
tunnel,
|
||||
stream: Some(stream),
|
||||
task: Some(AbortOnDropHandle::new(task)),
|
||||
@@ -165,15 +189,22 @@ impl<T: Tunnel> MpscTunnel<T> {
|
||||
|
||||
pub fn new_direct(tunnel: T) -> Self {
|
||||
let (stream, sink) = tunnel.split();
|
||||
let info = tunnel.info();
|
||||
let batch_flush = info
|
||||
.as_ref()
|
||||
.map(|i| matches!(i.tunnel_type.as_str(), "ring" | "udp"))
|
||||
.unwrap_or(false);
|
||||
Self {
|
||||
tx: None,
|
||||
direct_sink: Some(Arc::new(SpinSink::new(sink))),
|
||||
direct_batch_flush: batch_flush,
|
||||
tunnel,
|
||||
stream: Some(stream),
|
||||
task: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "MpscTunnel"))]
|
||||
async fn forward_one_round(
|
||||
rx: &mut Receiver<ZCPacket>,
|
||||
sink: &mut Pin<Box<dyn ZCPacketSink>>,
|
||||
@@ -187,6 +218,7 @@ impl<T: Tunnel> MpscTunnel<T> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "MpscTunnel"))]
|
||||
async fn forward_one_round_no_timeout(
|
||||
rx: &mut Receiver<ZCPacket>,
|
||||
sink: &mut Pin<Box<dyn ZCPacketSink>>,
|
||||
@@ -204,6 +236,7 @@ impl<T: Tunnel> MpscTunnel<T> {
|
||||
sink.flush().await
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "MpscTunnel"))]
|
||||
async fn forward_one_round_with_timeout(
|
||||
rx: &mut Receiver<ZCPacket>,
|
||||
sink: &mut Pin<Box<dyn ZCPacketSink>>,
|
||||
@@ -235,6 +268,7 @@ impl<T: Tunnel> MpscTunnel<T> {
|
||||
MpscTunnelSender {
|
||||
channel_tx: self.tx.as_ref().cloned(),
|
||||
direct_sink: self.direct_sink.clone(),
|
||||
direct_batch_flush: self.direct_batch_flush,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,8 +337,7 @@ mod tests {
|
||||
for i in 0..1000000 {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
|
||||
let a = sink1
|
||||
.send_async(ZCPacket::new_with_payload("hello".as_bytes()))
|
||||
.await;
|
||||
.send_async(ZCPacket::new_with_payload("hello".as_bytes())).await;
|
||||
if a.is_err() {
|
||||
tracing::info!(?a, "t2 exit with err");
|
||||
break;
|
||||
@@ -323,8 +356,7 @@ mod tests {
|
||||
for i in 0..1000000 {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
let a = sink2
|
||||
.send_async(ZCPacket::new_with_payload("hello2".as_bytes()))
|
||||
.await;
|
||||
.send_async(ZCPacket::new_with_payload("hello2".as_bytes())).await;
|
||||
if a.is_err() {
|
||||
tracing::info!(?a, "t3 exit with err");
|
||||
break;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
use bytes::Buf;
|
||||
use bytes::Bytes;
|
||||
use bytes::BytesMut;
|
||||
use zerocopy::byteorder::*;
|
||||
use zerocopy::AsBytes;
|
||||
use zerocopy::FromBytes;
|
||||
use zerocopy::FromZeroes;
|
||||
use zerocopy::byteorder::*;
|
||||
|
||||
type DefaultEndian = LittleEndian;
|
||||
|
||||
@@ -46,8 +47,6 @@ pub struct V4HolePunchPacket {
|
||||
pub struct V6HolePunchPacket {
|
||||
pub dst_ipv6: [u8; 16],
|
||||
pub dst_port: U16<DefaultEndian>,
|
||||
pub preferred_src_ipv6: [u8; 16],
|
||||
pub preferred_src_ifindex: U32<DefaultEndian>,
|
||||
}
|
||||
|
||||
#[repr(C, packed)]
|
||||
@@ -485,8 +484,16 @@ impl ZCPacket {
|
||||
let payload_off = ret.packet_type.get_packet_offsets().payload_offset;
|
||||
let total_len = payload_off + payload.len();
|
||||
ret.inner.reserve(total_len);
|
||||
unsafe { ret.inner.set_len(total_len) };
|
||||
ret.mut_payload()[..payload.len()].copy_from_slice(payload);
|
||||
|
||||
// SAFETY: `reserve` guarantees capacity >= total_len.
|
||||
// We zero the header region and copy payload before advancing length,
|
||||
// so every byte in [0..total_len) is initialized before any read.
|
||||
unsafe {
|
||||
let ptr = ret.inner.as_mut_ptr();
|
||||
std::ptr::write_bytes(ptr, 0, payload_off);
|
||||
std::ptr::copy_nonoverlapping(payload.as_ptr(), ptr.add(payload_off), payload.len());
|
||||
ret.inner.set_len(total_len);
|
||||
}
|
||||
ret
|
||||
}
|
||||
|
||||
@@ -494,12 +501,12 @@ impl ZCPacket {
|
||||
let mut ret = Self::new_nic_packet();
|
||||
ret.inner.reserve(cap);
|
||||
let total_len = ret.packet_type.get_packet_offsets().payload_offset - packet_info_len;
|
||||
unsafe { ret.inner.set_len(total_len) };
|
||||
ret.inner.resize(total_len, 0);
|
||||
ret
|
||||
}
|
||||
|
||||
pub fn new_for_foreign_network(
|
||||
network_name: &String,
|
||||
network_name: &str,
|
||||
dst_peer_id: u32,
|
||||
foreign_zc_packet: &ZCPacket,
|
||||
) -> Self {
|
||||
@@ -508,26 +515,71 @@ impl ZCPacket {
|
||||
foreign_network_hdr.get_header_len() + foreign_zc_packet.tunnel_payload().len();
|
||||
|
||||
let mut ret = Self::new_nic_packet();
|
||||
let payload_off = ret.packet_type.get_packet_offsets().payload_offset;
|
||||
ret.inner.reserve(payload_off + total_payload_len);
|
||||
unsafe { ret.inner.set_len(payload_off + total_payload_len) };
|
||||
let offsets = ret.packet_type.get_packet_offsets();
|
||||
let payload_off = offsets.payload_offset;
|
||||
let pm_hdr_off = offsets.peer_manager_header_offset;
|
||||
let total_len = payload_off + total_payload_len;
|
||||
ret.inner.reserve(total_len);
|
||||
|
||||
let fixed_hdr_len = std::mem::size_of::<ForeignNetworkPacketHeader>();
|
||||
ret.mut_payload()[..fixed_hdr_len].copy_from_slice(foreign_network_hdr.as_bytes());
|
||||
|
||||
let name_offset = foreign_network_hdr.network_name_offset.get() as usize;
|
||||
let name_len = foreign_network_hdr.network_name_len.get() as usize;
|
||||
ret.mut_payload()[name_offset..name_offset + name_len]
|
||||
.copy_from_slice(network_name.as_bytes());
|
||||
let foreign_payload = foreign_zc_packet.tunnel_payload();
|
||||
|
||||
ret.mut_payload()[foreign_network_hdr.get_header_len()..]
|
||||
.copy_from_slice(foreign_zc_packet.tunnel_payload());
|
||||
// Construct the PeerManagerHeader on the stack so we can write it
|
||||
// directly into the buffer, avoiding a separate mut_peer_manager_header()
|
||||
// call after set_len.
|
||||
let pm_hdr = PeerManagerHeader {
|
||||
from_peer_id: 0.into(),
|
||||
to_peer_id: 0.into(),
|
||||
packet_type: PacketType::ForeignNetworkPacket as u8,
|
||||
flags: 0,
|
||||
forward_counter: 0,
|
||||
reserved: 0,
|
||||
len: U32::new(total_payload_len as u32),
|
||||
};
|
||||
|
||||
let hdr = ret.mut_peer_manager_header().unwrap();
|
||||
hdr.from_peer_id = 0.into();
|
||||
hdr.to_peer_id = 0.into();
|
||||
hdr.packet_type = PacketType::ForeignNetworkPacket as u8;
|
||||
hdr.len.set(total_payload_len as u32);
|
||||
// SAFETY: `reserve` guarantees capacity >= total_len.
|
||||
// We zero only the tunnel-header reserved space [0..pm_hdr_off], write
|
||||
// the PeerManagerHeader directly at pm_hdr_off, then copy the foreign
|
||||
// network header, network name, and payload. Every byte in [0..total_len)
|
||||
// is initialized before set_len.
|
||||
unsafe {
|
||||
let ptr = ret.inner.as_mut_ptr();
|
||||
|
||||
// Zero the tunnel header reserved space only (not the PM header region)
|
||||
std::ptr::write_bytes(ptr, 0, pm_hdr_off);
|
||||
|
||||
// Write PeerManagerHeader directly
|
||||
std::ptr::copy_nonoverlapping(
|
||||
pm_hdr.as_bytes().as_ptr(),
|
||||
ptr.add(pm_hdr_off),
|
||||
std::mem::size_of::<PeerManagerHeader>(),
|
||||
);
|
||||
|
||||
// Copy foreign network fixed header
|
||||
std::ptr::copy_nonoverlapping(
|
||||
foreign_network_hdr.as_bytes().as_ptr(),
|
||||
ptr.add(payload_off),
|
||||
fixed_hdr_len,
|
||||
);
|
||||
|
||||
// Copy network name
|
||||
std::ptr::copy_nonoverlapping(
|
||||
network_name.as_ptr(),
|
||||
ptr.add(payload_off + name_offset),
|
||||
name_len,
|
||||
);
|
||||
|
||||
// Copy foreign payload
|
||||
std::ptr::copy_nonoverlapping(
|
||||
foreign_payload.as_ptr(),
|
||||
ptr.add(payload_off + foreign_network_hdr.get_header_len()),
|
||||
foreign_payload.len(),
|
||||
);
|
||||
|
||||
ret.inner.set_len(total_len);
|
||||
}
|
||||
|
||||
ret
|
||||
}
|
||||
@@ -587,7 +639,8 @@ impl ZCPacket {
|
||||
}
|
||||
|
||||
pub fn payload_bytes(mut self) -> BytesMut {
|
||||
self.inner.split_off(self.payload_offset())
|
||||
self.inner.advance(self.payload_offset());
|
||||
self.inner
|
||||
}
|
||||
|
||||
pub fn peer_manager_header(&self) -> Option<&PeerManagerHeader> {
|
||||
@@ -652,11 +705,12 @@ impl ZCPacket {
|
||||
}
|
||||
|
||||
pub fn tunnel_payload_bytes(mut self) -> BytesMut {
|
||||
self.inner.split_off(
|
||||
self.inner.advance(
|
||||
self.packet_type
|
||||
.get_packet_offsets()
|
||||
.peer_manager_header_offset,
|
||||
)
|
||||
);
|
||||
self.inner
|
||||
}
|
||||
|
||||
pub fn convert_type(mut self, target_packet_type: ZCPacketType) -> Self {
|
||||
@@ -697,12 +751,13 @@ impl ZCPacket {
|
||||
.get_packet_offsets()
|
||||
.peer_manager_header_offset;
|
||||
let mut buf = BytesMut::with_capacity(new_pm_offset + tunnel_payload.len());
|
||||
unsafe { buf.set_len(new_pm_offset) };
|
||||
buf.resize(new_pm_offset, 0);
|
||||
buf.extend_from_slice(tunnel_payload);
|
||||
return Self::new_from_buf(buf, target_packet_type);
|
||||
}
|
||||
|
||||
Self::new_from_buf(self.inner.split_off(new_offset), target_packet_type)
|
||||
self.inner.advance(new_offset);
|
||||
Self::new_from_buf(self.inner, target_packet_type)
|
||||
}
|
||||
|
||||
pub fn into_bytes(self) -> Bytes {
|
||||
|
||||
@@ -11,7 +11,7 @@ use async_trait::async_trait;
|
||||
use futures::{Sink, SinkExt, Stream, StreamExt};
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel};
|
||||
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -130,6 +130,7 @@ impl RingSink {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure_all)]
|
||||
impl Sink<SinkItem> for RingSink {
|
||||
type Error = SinkError;
|
||||
|
||||
@@ -196,7 +197,8 @@ pub struct RingTunnelListener {
|
||||
|
||||
impl RingTunnelListener {
|
||||
pub fn new(key: url::Url) -> Self {
|
||||
let (conn_sender, conn_receiver) = unbounded_channel();
|
||||
let (conn_sender, conn_receiver) =
|
||||
hotpath::channel!(tokio::sync::mpsc::unbounded_channel());
|
||||
RingTunnelListener {
|
||||
listener_addr: key,
|
||||
conn_sender,
|
||||
|
||||
+176
-276
@@ -2,22 +2,19 @@ use std::{
|
||||
fmt::Debug,
|
||||
net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6},
|
||||
sync::{Arc, Weak},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use anyhow::Context;
|
||||
use async_trait::async_trait;
|
||||
use bytes::BytesMut;
|
||||
use dashmap::DashMap;
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use futures::{SinkExt, StreamExt, stream::FuturesUnordered};
|
||||
use rand::{Rng, SeedableRng};
|
||||
use zerocopy::{AsBytes, FromBytes};
|
||||
|
||||
use tokio::{
|
||||
net::UdpSocket,
|
||||
sync::mpsc::{
|
||||
Receiver, Sender, UnboundedReceiver, UnboundedSender, channel, unbounded_channel,
|
||||
},
|
||||
sync::mpsc::{Receiver, Sender, UnboundedReceiver, UnboundedSender},
|
||||
task::JoinSet,
|
||||
};
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
@@ -38,7 +35,6 @@ use crate::{
|
||||
common::{TunnelWrapper, reserve_buf},
|
||||
packet_def::{UdpPacketType, ZCPacket, ZCPacketType},
|
||||
ring::RingTunnel,
|
||||
udp_src,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -47,12 +43,6 @@ pub const UDP_DATA_MTU: usize = 2000;
|
||||
type UdpCloseEventSender = UnboundedSender<(SocketAddr, Option<TunnelError>)>;
|
||||
type UdpCloseEventReceiver = UnboundedReceiver<(SocketAddr, Option<TunnelError>)>;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct PreferredIpv6Source {
|
||||
pub ip: Ipv6Addr,
|
||||
pub ifindex: u32,
|
||||
}
|
||||
|
||||
fn new_udp_packet<F>(f: F, udp_body: Option<&[u8]>) -> ZCPacket
|
||||
where
|
||||
F: FnOnce(&mut UDPTunnelHeader),
|
||||
@@ -107,18 +97,11 @@ pub fn new_hole_punch_packet(tid: u32, buf_len: u16) -> ZCPacket {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn new_v6_hole_punch_packet(
|
||||
dst: &SocketAddrV6,
|
||||
preferred_src: Option<PreferredIpv6Source>,
|
||||
) -> ZCPacket {
|
||||
pub fn new_v6_hole_punch_packet(dst: &SocketAddrV6) -> ZCPacket {
|
||||
// generate a 128 bytes vec with random data
|
||||
let mut body = V6HolePunchPacket::default();
|
||||
body.dst_ipv6.copy_from_slice(&dst.ip().octets());
|
||||
body.dst_port.set(dst.port());
|
||||
if let Some(src) = preferred_src {
|
||||
body.preferred_src_ipv6.copy_from_slice(&src.ip.octets());
|
||||
body.preferred_src_ifindex.set(src.ifindex);
|
||||
}
|
||||
new_udp_packet(
|
||||
|header| {
|
||||
header.msg_type = UdpPacketType::V6HolePunch as u8;
|
||||
@@ -153,18 +136,10 @@ fn extract_dst_addr_from_v4_hole_punch_packet(buf: &[u8]) -> Option<SocketAddrV4
|
||||
Some(SocketAddrV4::new(ip, body.dst_port.get()))
|
||||
}
|
||||
|
||||
fn extract_v6_hole_punch_packet(buf: &[u8]) -> Option<(SocketAddrV6, Option<PreferredIpv6Source>)> {
|
||||
fn extrace_dst_addr_from_hole_punch_packet(buf: &[u8]) -> Option<SocketAddrV6> {
|
||||
let body = V6HolePunchPacket::ref_from_prefix(buf)?;
|
||||
let ip = Ipv6Addr::from(body.dst_ipv6);
|
||||
let preferred_src_ipv6 = Ipv6Addr::from(body.preferred_src_ipv6);
|
||||
let preferred_src = (!preferred_src_ipv6.is_unspecified()).then_some(PreferredIpv6Source {
|
||||
ip: preferred_src_ipv6,
|
||||
ifindex: body.preferred_src_ifindex.get(),
|
||||
});
|
||||
Some((
|
||||
SocketAddrV6::new(ip, body.dst_port.get(), 0, 0),
|
||||
preferred_src,
|
||||
))
|
||||
Some(SocketAddrV6::new(ip, body.dst_port.get(), 0, 0))
|
||||
}
|
||||
|
||||
fn is_stun_packet(b: &[u8]) -> bool {
|
||||
@@ -177,10 +152,9 @@ fn is_stun_packet(b: &[u8]) -> bool {
|
||||
pub async fn send_v6_hole_punch_packet(
|
||||
listener_port: u16,
|
||||
dst_addr: SocketAddrV6,
|
||||
preferred_src: Option<PreferredIpv6Source>,
|
||||
) -> Result<(), TunnelError> {
|
||||
let local_socket = UdpSocket::bind("[::1]:0").await?;
|
||||
let udp_packet = new_v6_hole_punch_packet(&dst_addr, preferred_src);
|
||||
let udp_packet = new_v6_hole_punch_packet(&dst_addr);
|
||||
let remote_addr = format!("[::1]:{}", listener_port)
|
||||
.parse::<SocketAddr>()
|
||||
.unwrap();
|
||||
@@ -292,14 +266,151 @@ fn get_zcpacket_from_buf(buf: BytesMut, allow_stun: bool) -> Result<ZCPacket, Tu
|
||||
Ok(zc_packet)
|
||||
}
|
||||
|
||||
#[instrument]
|
||||
#[cfg(feature = "udp-gso")]
|
||||
const UDP_BATCH_SIZE: usize = 8;
|
||||
|
||||
#[cfg(feature = "udp-gso")]
|
||||
async fn forward_from_ring_to_udp(
|
||||
mut ring_recv: RingStream,
|
||||
socket: &Arc<UdpSocket>,
|
||||
addr: &SocketAddr,
|
||||
conn_id: u32,
|
||||
) -> Option<TunnelError> {
|
||||
tracing::debug!("udp forward from ring to udp");
|
||||
use quinn_udp::{Transmit, UdpSockRef, UdpSocketState};
|
||||
use std::io;
|
||||
|
||||
tracing::debug!("udp forward from ring to udp (GSO batch)");
|
||||
|
||||
let udp_state = match UdpSocketState::new(UdpSockRef::from(&**socket)) {
|
||||
Ok(state) => state,
|
||||
Err(e) => {
|
||||
tracing::warn!(?e, "failed to init UdpSocketState, fallback to per-packet send");
|
||||
return forward_from_ring_to_udp_fallback(ring_recv, socket, addr, conn_id).await;
|
||||
}
|
||||
};
|
||||
let max_gso = udp_state.max_gso_segments();
|
||||
tracing::info!(max_gso, "udp GSO segments supported");
|
||||
|
||||
loop {
|
||||
// 1. Get first packet (await)
|
||||
let first = match ring_recv.next().await {
|
||||
Some(Ok(pkt)) => convert_to_udp_bytes(pkt, conn_id),
|
||||
Some(Err(e)) => return Some(e),
|
||||
None => return None,
|
||||
};
|
||||
|
||||
// 2. Try to drain more packets (non-blocking via poll_next with noop waker)
|
||||
use std::pin::Pin;
|
||||
use std::task::{Poll, Context};
|
||||
use futures::task::noop_waker;
|
||||
use futures::Stream;
|
||||
let mut batch: Vec<bytes::Bytes> = vec![first];
|
||||
while batch.len() < UDP_BATCH_SIZE.min(max_gso) {
|
||||
let waker = noop_waker();
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
match Pin::new(&mut ring_recv).poll_next(&mut cx) {
|
||||
Poll::Ready(Some(Ok(pkt))) => {
|
||||
batch.push(convert_to_udp_bytes(pkt, conn_id));
|
||||
}
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Check if all same size (GSO requirement)
|
||||
let seg_size = batch[0].len();
|
||||
let all_same = batch.iter().all(|b| b.len() == seg_size);
|
||||
|
||||
if batch.len() == 1 || !all_same || max_gso == 1 {
|
||||
// Fallback: send individually
|
||||
for buf in &batch {
|
||||
if let Err(e) = send_one(&udp_state, socket, addr, buf).await {
|
||||
return Some(TunnelError::IOError(e));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// GSO batch: concatenate + single sendmsg
|
||||
let mut contents = Vec::with_capacity(seg_size * batch.len());
|
||||
for buf in &batch {
|
||||
contents.extend_from_slice(buf);
|
||||
}
|
||||
let transmit = Transmit {
|
||||
destination: *addr,
|
||||
ecn: None,
|
||||
contents: &contents,
|
||||
segment_size: Some(seg_size),
|
||||
src_ip: None,
|
||||
};
|
||||
if let Err(e) = send_one_gso(&udp_state, socket, &transmit).await {
|
||||
return Some(TunnelError::IOError(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "udp-gso")]
|
||||
fn convert_to_udp_bytes(mut packet: ZCPacket, conn_id: u32) -> bytes::Bytes {
|
||||
let mut packet = packet.convert_type(ZCPacketType::UDP);
|
||||
let udp_payload_len = packet.udp_payload().len();
|
||||
let header = packet.mut_udp_tunnel_header().unwrap();
|
||||
header.conn_id.set(conn_id);
|
||||
header.len.set(udp_payload_len as u16);
|
||||
header.msg_type = UdpPacketType::Data as u8;
|
||||
packet.into_bytes()
|
||||
}
|
||||
|
||||
#[cfg(feature = "udp-gso")]
|
||||
async fn send_one(
|
||||
udp_state: &quinn_udp::UdpSocketState,
|
||||
socket: &Arc<UdpSocket>,
|
||||
addr: &SocketAddr,
|
||||
buf: &[u8],
|
||||
) -> Result<(), std::io::Error> {
|
||||
use quinn_udp::{Transmit, UdpSockRef};
|
||||
let transmit = Transmit {
|
||||
destination: *addr,
|
||||
ecn: None,
|
||||
contents: buf,
|
||||
segment_size: None,
|
||||
src_ip: None,
|
||||
};
|
||||
send_one_gso(udp_state, socket, &transmit).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "udp-gso")]
|
||||
async fn send_one_gso(
|
||||
udp_state: &quinn_udp::UdpSocketState,
|
||||
socket: &Arc<UdpSocket>,
|
||||
transmit: &quinn_udp::Transmit<'_>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
use quinn_udp::UdpSockRef;
|
||||
loop {
|
||||
match udp_state.send(UdpSockRef::from(&**socket), transmit) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "udp-gso"))]
|
||||
async fn forward_from_ring_to_udp(
|
||||
mut ring_recv: RingStream,
|
||||
socket: &Arc<UdpSocket>,
|
||||
addr: &SocketAddr,
|
||||
conn_id: u32,
|
||||
) -> Option<TunnelError> {
|
||||
forward_from_ring_to_udp_fallback(ring_recv, socket, addr, conn_id).await
|
||||
}
|
||||
|
||||
async fn forward_from_ring_to_udp_fallback(
|
||||
mut ring_recv: RingStream,
|
||||
socket: &Arc<UdpSocket>,
|
||||
addr: &SocketAddr,
|
||||
conn_id: u32,
|
||||
) -> Option<TunnelError> {
|
||||
tracing::debug!("udp forward from ring to udp (per-packet)");
|
||||
loop {
|
||||
let buf = ring_recv.next().await?;
|
||||
let packet = match buf {
|
||||
@@ -317,7 +428,6 @@ async fn forward_from_ring_to_udp(
|
||||
header.msg_type = UdpPacketType::Data as u8;
|
||||
|
||||
let buf = packet.into_bytes();
|
||||
tracing::trace!(?udp_payload_len, ?buf, "udp forward from ring to udp");
|
||||
let ret = socket.send_to(&buf, &addr).await;
|
||||
if ret.is_err() {
|
||||
return Some(TunnelError::IOError(ret.unwrap_err()));
|
||||
@@ -395,7 +505,10 @@ impl UdpConnection {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn handle_packet_from_remote(&mut self, zc_packet: ZCPacket) -> Result<(), TunnelError> {
|
||||
pub async fn handle_packet_from_remote(
|
||||
&mut self,
|
||||
zc_packet: ZCPacket,
|
||||
) -> Result<(), TunnelError> {
|
||||
let header = zc_packet.udp_tunnel_header().unwrap();
|
||||
let conn_id = header.conn_id.get();
|
||||
|
||||
@@ -407,14 +520,7 @@ impl UdpConnection {
|
||||
return Err(TunnelError::ConnIdNotMatch(self.conn_id, conn_id));
|
||||
}
|
||||
|
||||
if zc_packet.is_lossy() {
|
||||
if let Err(e) = self.ring_sender.try_send(zc_packet) {
|
||||
tracing::trace!(?e, "ring sender full, drop lossy packet");
|
||||
}
|
||||
} else if self.ring_sender.force_send(zc_packet).is_err() {
|
||||
tracing::trace!("ring sender full, reject non-lossy packet");
|
||||
return Err(TunnelError::BufferFull);
|
||||
}
|
||||
self.ring_sender.send(zc_packet).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -461,15 +567,8 @@ impl UdpTunnelListenerData {
|
||||
let socket = self.socket.as_ref().unwrap().clone();
|
||||
|
||||
let sack_buf = new_sack_packet(conn_id, magic).into_bytes();
|
||||
if self
|
||||
.sock_map
|
||||
.get(&remote_addr)
|
||||
.is_some_and(|conn| conn.conn_id == conn_id)
|
||||
{
|
||||
if let Err(e) = socket.send_to(&sack_buf, remote_addr).await {
|
||||
tracing::error!(?e, "udp resend sack packet error");
|
||||
}
|
||||
tracing::debug!(?conn_id, ?remote_addr, "udp duplicate syn, resent sack");
|
||||
if let Err(e) = socket.send_to(&sack_buf, remote_addr).await {
|
||||
tracing::error!(?e, "udp send sack packet error");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -481,43 +580,15 @@ impl UdpTunnelListenerData {
|
||||
"udp build tunnel for listener"
|
||||
);
|
||||
|
||||
let new_internal_conn = || {
|
||||
UdpConnection::new(
|
||||
socket.clone(),
|
||||
conn_id,
|
||||
remote_addr,
|
||||
RingSink::new(ring_for_recv_udp.clone()),
|
||||
RingStream::new(ring_for_send_udp.clone()),
|
||||
self.close_event_sender.clone(),
|
||||
)
|
||||
};
|
||||
let duplicate_syn = match self.sock_map.entry(remote_addr) {
|
||||
dashmap::mapref::entry::Entry::Occupied(entry) if entry.get().conn_id == conn_id => {
|
||||
true
|
||||
}
|
||||
dashmap::mapref::entry::Entry::Occupied(mut entry) => {
|
||||
entry.insert(new_internal_conn());
|
||||
false
|
||||
}
|
||||
dashmap::mapref::entry::Entry::Vacant(entry) => {
|
||||
entry.insert(new_internal_conn());
|
||||
false
|
||||
}
|
||||
};
|
||||
if duplicate_syn {
|
||||
if let Err(e) = socket.send_to(&sack_buf, remote_addr).await {
|
||||
tracing::error!(?e, "udp resend sack packet error");
|
||||
}
|
||||
tracing::debug!(?conn_id, ?remote_addr, "udp duplicate syn, resent sack");
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = socket.send_to(&sack_buf, remote_addr).await {
|
||||
self.sock_map
|
||||
.remove_if(&remote_addr, |_, conn| conn.conn_id == conn_id);
|
||||
tracing::error!(?e, "udp send sack packet error");
|
||||
return;
|
||||
}
|
||||
let internal_conn = UdpConnection::new(
|
||||
socket.clone(),
|
||||
conn_id,
|
||||
remote_addr,
|
||||
RingSink::new(ring_for_recv_udp.clone()),
|
||||
RingStream::new(ring_for_send_udp.clone()),
|
||||
self.close_event_sender.clone(),
|
||||
);
|
||||
self.sock_map.insert(remote_addr, internal_conn);
|
||||
|
||||
let conn = Box::new(TunnelWrapper::new(
|
||||
Box::new(RingStream::new(ring_for_recv_udp)),
|
||||
@@ -541,7 +612,7 @@ impl UdpTunnelListenerData {
|
||||
}
|
||||
}
|
||||
|
||||
fn do_forward_one_packet_to_conn(&self, zc_packet: ZCPacket, addr: SocketAddr) {
|
||||
async fn do_forward_one_packet_to_conn(&self, zc_packet: ZCPacket, addr: SocketAddr) {
|
||||
let header = zc_packet.udp_tunnel_header().unwrap();
|
||||
if header.msg_type == UdpPacketType::Syn as u8 {
|
||||
tokio::spawn(Self::handle_new_connect(self.clone(), addr, zc_packet));
|
||||
@@ -585,61 +656,23 @@ impl UdpTunnelListenerData {
|
||||
tracing::warn!(?addr, "v6 hole punch packet should be sent from ipv6");
|
||||
return;
|
||||
}
|
||||
let Some((dst_addr, preferred_src)) =
|
||||
extract_v6_hole_punch_packet(zc_packet.udp_payload())
|
||||
let Some(dst_addr) = extrace_dst_addr_from_hole_punch_packet(zc_packet.udp_payload())
|
||||
else {
|
||||
tracing::warn!("invalid v6 hole punch packet");
|
||||
return;
|
||||
};
|
||||
let socket = self.socket.as_ref().unwrap().clone();
|
||||
let udp_packet = new_hole_punch_packet(1, 32);
|
||||
let udp_packet = udp_packet.into_bytes();
|
||||
let sent_with_src = if let Some(src) = preferred_src {
|
||||
match udp_src::send_to_with_src_ipv6(
|
||||
&socket,
|
||||
src.ip,
|
||||
src.ifindex,
|
||||
dst_addr,
|
||||
&udp_packet,
|
||||
) {
|
||||
Ok(ret) => {
|
||||
tracing::debug!(
|
||||
?src,
|
||||
?dst_addr,
|
||||
?ret,
|
||||
"udp forward packet send hole punch packet with preferred ipv6 source"
|
||||
);
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
?src,
|
||||
?dst_addr,
|
||||
?e,
|
||||
"udp forward packet preferred ipv6 source failed, falling back"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if !sent_with_src
|
||||
&& let Err(e) = socket.try_send_to(&udp_packet, SocketAddr::V6(dst_addr))
|
||||
{
|
||||
if let Err(e) = socket.try_send_to(&udp_packet.into_bytes(), SocketAddr::V6(dst_addr)) {
|
||||
tracing::error!(?e, "udp send hole punch packet error");
|
||||
}
|
||||
tracing::debug!(
|
||||
?dst_addr,
|
||||
?preferred_src,
|
||||
"udp forward packet send hole punch packet"
|
||||
);
|
||||
tracing::debug!(?dst_addr, "udp forward packet send hole punch packet");
|
||||
} else if header.msg_type != UdpPacketType::HolePunch as u8 {
|
||||
let Some(mut conn) = self.sock_map.get_mut(&addr) else {
|
||||
tracing::trace!(?header, "udp forward packet error, connection not found");
|
||||
return;
|
||||
};
|
||||
if let Err(e) = conn.handle_packet_from_remote(zc_packet) {
|
||||
if let Err(e) = conn.handle_packet_from_remote(zc_packet).await {
|
||||
tracing::trace!(?e, "udp forward packet error");
|
||||
}
|
||||
} else {
|
||||
@@ -652,7 +685,7 @@ impl UdpTunnelListenerData {
|
||||
let mut buf = BytesMut::new();
|
||||
loop {
|
||||
match udp_recv_from_socket_forward_task(&socket, &mut buf, true).await {
|
||||
Ok((zc_packet, addr)) => self.do_forward_one_packet_to_conn(zc_packet, addr),
|
||||
Ok((zc_packet, addr)) => self.do_forward_one_packet_to_conn(zc_packet, addr).await,
|
||||
Err(e) => {
|
||||
tracing::error!(?e, "udp recv packet error");
|
||||
break;
|
||||
@@ -675,8 +708,9 @@ pub struct UdpTunnelListener {
|
||||
|
||||
impl UdpTunnelListener {
|
||||
pub fn new(addr: url::Url) -> Self {
|
||||
let (close_event_send, close_event_recv) = unbounded_channel();
|
||||
let (conn_send, conn_recv) = channel(100);
|
||||
let (close_event_send, close_event_recv) =
|
||||
hotpath::channel!(tokio::sync::mpsc::unbounded_channel());
|
||||
let (conn_send, conn_recv) = hotpath::channel!(tokio::sync::mpsc::channel(100));
|
||||
Self {
|
||||
addr: addr.clone(),
|
||||
socket: None,
|
||||
@@ -807,13 +841,6 @@ impl UdpTunnelConnector {
|
||||
}
|
||||
}
|
||||
|
||||
fn should_resend_syn_to_hole_punch_source(
|
||||
recv_addr: SocketAddr,
|
||||
expected_addr: SocketAddr,
|
||||
) -> bool {
|
||||
recv_addr == expected_addr
|
||||
}
|
||||
|
||||
async fn wait_sack(
|
||||
socket: &UdpSocket,
|
||||
addr: SocketAddr,
|
||||
@@ -829,34 +856,12 @@ impl UdpTunnelConnector {
|
||||
)
|
||||
.await??;
|
||||
let zc_packet = get_zcpacket_from_buf(buf.split(), false)?;
|
||||
let header = zc_packet.udp_tunnel_header().unwrap();
|
||||
if header.msg_type == UdpPacketType::HolePunch as u8 {
|
||||
tracing::debug!(?recv_addr, ?addr, "udp wait sack got hole punch packet");
|
||||
if Self::should_resend_syn_to_hole_punch_source(recv_addr, addr) {
|
||||
let udp_packet = new_syn_packet(conn_id, magic).into_bytes();
|
||||
match socket.send_to(&udp_packet, recv_addr).await {
|
||||
Ok(ret) => {
|
||||
tracing::debug!(?recv_addr, ?ret, "udp send syn to hole punch source")
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(?recv_addr, ?e, "udp send syn to hole punch source failed")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tracing::debug!(
|
||||
?recv_addr,
|
||||
?addr,
|
||||
"ignore hole punch packet from unexpected source"
|
||||
);
|
||||
}
|
||||
return Err(TunnelError::InvalidPacket(
|
||||
"got hole punch packet while waiting for sack".to_owned(),
|
||||
));
|
||||
}
|
||||
if recv_addr != addr {
|
||||
tracing::warn!(?recv_addr, ?addr, ?usize, "udp wait sack addr not match");
|
||||
}
|
||||
|
||||
let header = zc_packet.udp_tunnel_header().unwrap();
|
||||
|
||||
if header.conn_id.get() != conn_id {
|
||||
return Err(super::TunnelError::ConnIdNotMatch(
|
||||
header.conn_id.get(),
|
||||
@@ -916,7 +921,8 @@ impl UdpTunnelConnector {
|
||||
"udp build tunnel for connector"
|
||||
);
|
||||
|
||||
let (close_event_sender, mut close_event_recv) = unbounded_channel();
|
||||
let (close_event_sender, mut close_event_recv) =
|
||||
hotpath::channel!(tokio::sync::mpsc::unbounded_channel());
|
||||
|
||||
let ring_recv = RingStream::new(ring_for_send_udp.clone());
|
||||
let ring_sender = RingSink::new(ring_for_recv_udp.clone());
|
||||
@@ -937,7 +943,7 @@ impl UdpTunnelConnector {
|
||||
match udp_recv_from_socket_forward_task(&socket_clone, &mut buf, false).await {
|
||||
Ok((zc_packet, addr)) => {
|
||||
tracing::trace!(?addr, "connector udp forward task done");
|
||||
if let Err(e) = udp_conn.handle_packet_from_remote(zc_packet) {
|
||||
if let Err(e) = udp_conn.handle_packet_from_remote(zc_packet).await {
|
||||
tracing::trace!(?e, ?addr, "udp forward packet error");
|
||||
}
|
||||
}
|
||||
@@ -998,23 +1004,6 @@ impl UdpTunnelConnector {
|
||||
let udp_packet = new_syn_packet(conn_id, magic).into_bytes();
|
||||
let ret = socket.send_to(&udp_packet, &addr).await?;
|
||||
tracing::warn!(?udp_packet, ?ret, "udp send syn");
|
||||
let resend_task = AbortOnDropHandle::new(tokio::spawn({
|
||||
let socket = socket.clone();
|
||||
let udp_packet = udp_packet.clone();
|
||||
let resend_addr = addr;
|
||||
async move {
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
match socket.send_to(&udp_packet, &resend_addr).await {
|
||||
Ok(ret) => tracing::trace!(?ret, ?resend_addr, "udp resend syn"),
|
||||
Err(e) => {
|
||||
tracing::debug!(?e, ?resend_addr, "udp resend syn failed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
// wait sack
|
||||
let recv_addr = tokio::time::timeout(
|
||||
@@ -1022,13 +1011,12 @@ impl UdpTunnelConnector {
|
||||
Self::wait_sack_loop(&socket, addr, conn_id, magic),
|
||||
)
|
||||
.await??;
|
||||
drop(resend_task);
|
||||
|
||||
if recv_addr != addr {
|
||||
tracing::debug!(?recv_addr, ?addr, "udp connect addr not match");
|
||||
}
|
||||
|
||||
self.build_tunnel(socket, recv_addr, conn_id).await
|
||||
self.build_tunnel(socket, addr, conn_id).await
|
||||
}
|
||||
|
||||
async fn connect_with_default_bind(
|
||||
@@ -1127,43 +1115,9 @@ mod tests {
|
||||
get_interface_name_by_ip,
|
||||
tests::{_tunnel_bench, _tunnel_echo_server, _tunnel_pingpong, wait_for_condition},
|
||||
},
|
||||
packet_def::PacketType,
|
||||
},
|
||||
};
|
||||
|
||||
fn new_udp_data_packet(conn_id: u32, packet_type: PacketType) -> ZCPacket {
|
||||
let mut packet = ZCPacket::new_with_payload(b"udp-data").convert_type(ZCPacketType::UDP);
|
||||
packet.fill_peer_manager_hdr(1, 2, packet_type as u8);
|
||||
let udp_payload_len = packet.udp_payload().len();
|
||||
let header = packet.mut_udp_tunnel_header().unwrap();
|
||||
header.conn_id.set(conn_id);
|
||||
header.msg_type = UdpPacketType::Data as u8;
|
||||
header.len.set(udp_payload_len as u16);
|
||||
packet
|
||||
}
|
||||
|
||||
fn assert_sync_packet_handler(_: fn(&mut UdpConnection, ZCPacket) -> Result<(), TunnelError>) {}
|
||||
|
||||
#[test]
|
||||
fn hole_punch_source_must_match_connect_addr_before_syn_resend() {
|
||||
let expected_addr: SocketAddr = "198.51.100.10:11010".parse().unwrap();
|
||||
let same_port_different_ip: SocketAddr = "198.51.100.11:11010".parse().unwrap();
|
||||
let same_ip_different_port: SocketAddr = "198.51.100.10:11011".parse().unwrap();
|
||||
|
||||
assert!(UdpTunnelConnector::should_resend_syn_to_hole_punch_source(
|
||||
expected_addr,
|
||||
expected_addr
|
||||
));
|
||||
assert!(!UdpTunnelConnector::should_resend_syn_to_hole_punch_source(
|
||||
same_port_different_ip,
|
||||
expected_addr
|
||||
));
|
||||
assert!(!UdpTunnelConnector::should_resend_syn_to_hole_punch_source(
|
||||
same_ip_different_port,
|
||||
expected_addr
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn udp_pingpong() {
|
||||
let listener = UdpTunnelListener::new("udp://0.0.0.0:5556".parse().unwrap());
|
||||
@@ -1171,43 +1125,6 @@ mod tests {
|
||||
_tunnel_pingpong(listener, connector).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn udp_connection_handler_uses_sync_nonblocking_ring_delivery() {
|
||||
assert_sync_packet_handler(UdpConnection::handle_packet_from_remote);
|
||||
|
||||
let socket = Arc::new(UdpSocket::bind("127.0.0.1:0").await.unwrap());
|
||||
let dst_addr = "127.0.0.1:1".parse().unwrap();
|
||||
let ring_for_send_udp = Arc::new(RingTunnel::new(8));
|
||||
let ring_for_recv_udp = Arc::new(RingTunnel::new(8));
|
||||
let (close_event_sender, _close_event_recv) = tokio::sync::mpsc::unbounded_channel();
|
||||
let mut conn = UdpConnection::new(
|
||||
socket,
|
||||
7,
|
||||
dst_addr,
|
||||
RingSink::new(ring_for_recv_udp),
|
||||
RingStream::new(ring_for_send_udp),
|
||||
close_event_sender,
|
||||
);
|
||||
|
||||
for _ in 0..16 {
|
||||
conn.handle_packet_from_remote(new_udp_data_packet(7, PacketType::Data))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut got_buffer_full = false;
|
||||
for _ in 0..16 {
|
||||
match conn.handle_packet_from_remote(new_udp_data_packet(7, PacketType::Ping)) {
|
||||
Ok(()) => {}
|
||||
Err(TunnelError::BufferFull) => {
|
||||
got_buffer_full = true;
|
||||
break;
|
||||
}
|
||||
Err(e) => panic!("unexpected error: {e:?}"),
|
||||
}
|
||||
}
|
||||
assert!(got_buffer_full);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn udp_bench() {
|
||||
let listener = UdpTunnelListener::new("udp://0.0.0.0:5555".parse().unwrap());
|
||||
@@ -1431,22 +1348,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v6_hole_punch_packet_preserves_preferred_source_ifindex() {
|
||||
let dst_addr = "[2001:db8::1]:10001".parse::<SocketAddrV6>().unwrap();
|
||||
let preferred_src = PreferredIpv6Source {
|
||||
ip: "2001:db8::2".parse().unwrap(),
|
||||
ifindex: 42,
|
||||
};
|
||||
|
||||
let packet = new_v6_hole_punch_packet(&dst_addr, Some(preferred_src));
|
||||
let (parsed_dst_addr, parsed_preferred_src) =
|
||||
extract_v6_hole_punch_packet(packet.udp_payload()).unwrap();
|
||||
|
||||
assert_eq!(parsed_dst_addr, dst_addr);
|
||||
assert_eq!(parsed_preferred_src, Some(preferred_src));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_v6_hole_punch_packet() {
|
||||
let mut lis = UdpTunnelListener::new("udp://[::]:0".parse().unwrap());
|
||||
@@ -1473,7 +1374,6 @@ mod tests {
|
||||
std::net::SocketAddr::V6(addr_v6) => addr_v6,
|
||||
_ => panic!("Expected an IPv6 address"),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
use std::{
|
||||
io,
|
||||
net::{Ipv6Addr, SocketAddrV6},
|
||||
};
|
||||
|
||||
use tokio::net::UdpSocket;
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn send_to_with_src_ipv6(
|
||||
socket: &UdpSocket,
|
||||
src_ip: Ipv6Addr,
|
||||
src_ifindex: u32,
|
||||
dst_addr: SocketAddrV6,
|
||||
buf: &[u8],
|
||||
) -> io::Result<usize> {
|
||||
#[cfg(target_env = "ohos")]
|
||||
{
|
||||
let _ = (socket, src_ip, src_ifindex, dst_addr, buf);
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"sending UDP with a selected IPv6 source is not supported on OHOS",
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(target_env = "ohos"))]
|
||||
{
|
||||
use std::{mem, os::fd::AsRawFd, ptr};
|
||||
|
||||
use nix::libc;
|
||||
|
||||
#[repr(align(8))]
|
||||
struct ControlBuffer([u8; 128]);
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
let ipi6_ifindex: libc::c_int = i32::try_from(src_ifindex).map_err(|_| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"IPv6 source interface index is out of range",
|
||||
)
|
||||
})?;
|
||||
#[cfg(not(target_os = "android"))]
|
||||
let ipi6_ifindex: libc::c_uint = src_ifindex;
|
||||
|
||||
let pktinfo = libc::in6_pktinfo {
|
||||
ipi6_addr: libc::in6_addr {
|
||||
s6_addr: src_ip.octets(),
|
||||
},
|
||||
ipi6_ifindex,
|
||||
};
|
||||
let mut iov = libc::iovec {
|
||||
iov_base: buf.as_ptr() as *mut libc::c_void,
|
||||
iov_len: buf.len(),
|
||||
};
|
||||
let dst_addr = socket2::SockAddr::from(std::net::SocketAddr::V6(dst_addr));
|
||||
let control_len = unsafe {
|
||||
libc::CMSG_SPACE(mem::size_of::<libc::in6_pktinfo>() as libc::c_uint) as usize
|
||||
};
|
||||
let mut control = ControlBuffer([0u8; 128]);
|
||||
if control_len > control.0.len() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"IPv6 packet info control buffer is too small",
|
||||
));
|
||||
}
|
||||
|
||||
let mut msg = unsafe { mem::zeroed::<libc::msghdr>() };
|
||||
msg.msg_name = dst_addr.as_ptr() as *mut libc::c_void;
|
||||
msg.msg_namelen = dst_addr.len() as _;
|
||||
msg.msg_iov = &mut iov;
|
||||
msg.msg_iovlen = 1;
|
||||
msg.msg_control = control.0.as_mut_ptr() as *mut libc::c_void;
|
||||
msg.msg_controllen = control_len as _;
|
||||
msg.msg_flags = 0;
|
||||
|
||||
unsafe {
|
||||
let cmsg = libc::CMSG_FIRSTHDR(&msg);
|
||||
if cmsg.is_null() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"IPv6 packet info control buffer is invalid",
|
||||
));
|
||||
}
|
||||
(*cmsg).cmsg_level = libc::IPPROTO_IPV6;
|
||||
(*cmsg).cmsg_type = libc::IPV6_PKTINFO;
|
||||
(*cmsg).cmsg_len =
|
||||
libc::CMSG_LEN(mem::size_of::<libc::in6_pktinfo>() as libc::c_uint) as _;
|
||||
ptr::write(libc::CMSG_DATA(cmsg) as *mut libc::in6_pktinfo, pktinfo);
|
||||
|
||||
let ret = libc::sendmsg(socket.as_raw_fd(), &msg, 0);
|
||||
if ret < 0 {
|
||||
Err(io::Error::last_os_error())
|
||||
} else {
|
||||
Ok(ret as usize)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn send_to_with_src_ipv6(
|
||||
socket: &UdpSocket,
|
||||
src_ip: Ipv6Addr,
|
||||
src_ifindex: u32,
|
||||
dst_addr: SocketAddrV6,
|
||||
buf: &[u8],
|
||||
) -> io::Result<usize> {
|
||||
use std::{mem, os::windows::io::AsRawSocket, ptr};
|
||||
|
||||
use windows::{
|
||||
Win32::Networking::WinSock::{
|
||||
CMSGHDR, IN6_ADDR, IN6_ADDR_0, IN6_PKTINFO, IPPROTO_IPV6, IPV6_PKTINFO, SOCKET,
|
||||
SOCKET_ERROR, WSABUF, WSAGetLastError, WSAMSG, WSASendMsg,
|
||||
},
|
||||
core::PSTR,
|
||||
};
|
||||
|
||||
fn cmsghdr_align(length: usize) -> usize {
|
||||
(length + mem::align_of::<CMSGHDR>() - 1) & !(mem::align_of::<CMSGHDR>() - 1)
|
||||
}
|
||||
|
||||
fn cmsgdata_align(length: usize) -> usize {
|
||||
(length + mem::align_of::<usize>() - 1) & !(mem::align_of::<usize>() - 1)
|
||||
}
|
||||
|
||||
fn cmsg_len(length: usize) -> usize {
|
||||
cmsgdata_align(mem::size_of::<CMSGHDR>()) + length
|
||||
}
|
||||
|
||||
fn cmsg_space(length: usize) -> usize {
|
||||
cmsgdata_align(mem::size_of::<CMSGHDR>() + cmsghdr_align(length))
|
||||
}
|
||||
|
||||
fn cmsg_data(cmsg: *mut CMSGHDR) -> *mut u8 {
|
||||
(cmsg as usize + cmsgdata_align(mem::size_of::<CMSGHDR>())) as *mut u8
|
||||
}
|
||||
|
||||
#[repr(align(8))]
|
||||
struct ControlBuffer([u8; 128]);
|
||||
|
||||
let dst = socket2::SockAddr::from(std::net::SocketAddr::V6(dst_addr));
|
||||
let mut data = WSABUF {
|
||||
len: buf.len() as u32,
|
||||
buf: PSTR(buf.as_ptr() as *mut u8),
|
||||
};
|
||||
let control_len = cmsg_space(mem::size_of::<IN6_PKTINFO>());
|
||||
let mut control = ControlBuffer([0u8; 128]);
|
||||
if control_len > control.0.len() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"IPv6 packet info control buffer is too small",
|
||||
));
|
||||
}
|
||||
let mut msg = WSAMSG {
|
||||
name: dst.as_ptr() as *mut _,
|
||||
namelen: dst.len(),
|
||||
lpBuffers: &mut data,
|
||||
dwBufferCount: 1,
|
||||
Control: WSABUF {
|
||||
len: control_len as u32,
|
||||
buf: PSTR(control.0.as_mut_ptr()),
|
||||
},
|
||||
dwFlags: 0,
|
||||
};
|
||||
|
||||
let pktinfo = IN6_PKTINFO {
|
||||
ipi6_addr: IN6_ADDR {
|
||||
u: IN6_ADDR_0 {
|
||||
Byte: src_ip.octets(),
|
||||
},
|
||||
},
|
||||
ipi6_ifindex: src_ifindex,
|
||||
};
|
||||
|
||||
unsafe {
|
||||
let cmsg = control.0.as_mut_ptr() as *mut CMSGHDR;
|
||||
(*cmsg).cmsg_level = IPPROTO_IPV6.0;
|
||||
(*cmsg).cmsg_type = IPV6_PKTINFO;
|
||||
(*cmsg).cmsg_len = cmsg_len(mem::size_of::<IN6_PKTINFO>());
|
||||
ptr::write(cmsg_data(cmsg) as *mut IN6_PKTINFO, pktinfo);
|
||||
msg.Control.len = control_len as u32;
|
||||
|
||||
let mut sent = 0;
|
||||
let ret = WSASendMsg(
|
||||
SOCKET(socket.as_raw_socket() as usize),
|
||||
&msg,
|
||||
0,
|
||||
Some(&mut sent),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
if ret == SOCKET_ERROR {
|
||||
return Err(io::Error::from_raw_os_error(WSAGetLastError().0));
|
||||
}
|
||||
Ok(sent as usize)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
pub(crate) fn send_to_with_src_ipv6(
|
||||
_socket: &UdpSocket,
|
||||
_src_ip: Ipv6Addr,
|
||||
_src_ifindex: u32,
|
||||
_dst_addr: SocketAddrV6,
|
||||
_buf: &[u8],
|
||||
) -> io::Result<usize> {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"sending UDP with a selected IPv6 source is not supported on this platform",
|
||||
))
|
||||
}
|
||||
@@ -6,7 +6,7 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use quanta::Instant;
|
||||
use hotpath::instant::Instant;
|
||||
|
||||
use super::{
|
||||
FromUrl, IpVersion, Tunnel, TunnelError, TunnelInfo, TunnelListener, TunnelUrl, ZCPacketSink,
|
||||
@@ -37,11 +37,7 @@ use crossbeam::atomic::AtomicCell;
|
||||
use dashmap::DashMap;
|
||||
use futures::{SinkExt, StreamExt, stream::FuturesUnordered};
|
||||
use rand::RngCore;
|
||||
use tokio::{
|
||||
net::UdpSocket,
|
||||
sync::{Mutex, mpsc::unbounded_channel},
|
||||
task::JoinSet,
|
||||
};
|
||||
use tokio::{net::UdpSocket, sync::Mutex, task::JoinSet};
|
||||
|
||||
const MAX_PACKET: usize = 2048;
|
||||
|
||||
@@ -474,7 +470,7 @@ pub struct WgTunnelListener {
|
||||
|
||||
impl WgTunnelListener {
|
||||
pub fn new(addr: url::Url, config: WgConfig) -> Self {
|
||||
let (conn_send, conn_recv) = unbounded_channel();
|
||||
let (conn_send, conn_recv) = hotpath::channel!(tokio::sync::mpsc::unbounded_channel());
|
||||
WgTunnelListener {
|
||||
addr,
|
||||
config,
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
# 计划 001:将共享 metrics/throughput 计数改为线程安全实现
|
||||
|
||||
> **执行者说明**:按步骤执行本计划。每一步都必须运行验证命令,并确认结果符合预期后再继续。如果触发“STOP 条件”中的任一情况,立即停止并报告,不要自行发挥。完成后更新 `plans/README.md` 中本计划的状态行,除非 reviewer 明确说明由他们维护索引。
|
||||
>
|
||||
> **漂移检查(首先运行)**:`git diff --stat 78146d16..HEAD -- easytier/src/common/stats_manager.rs easytier/src/tunnel/stats.rs easytier/src/tunnel/filter.rs easytier/src/proto/rpc_impl/server.rs easytier/src/tests`
|
||||
> 如果本计划写成后任何范围内文件发生变化,继续前必须对照“当前状态”中的摘录与实时代码;如果不匹配,按 STOP 条件处理。
|
||||
|
||||
## 状态
|
||||
|
||||
- **优先级**: P1
|
||||
- **工作量**: M
|
||||
- **风险**: MED
|
||||
- **依赖**: none
|
||||
- **类别**: bug
|
||||
- **计划生成于**: commit `78146d16`, 2026-06-18
|
||||
|
||||
## 为什么重要
|
||||
|
||||
核心 metrics 和 tunnel throughput 计数器当前用 `UnsafeCell<u64>` 保存,并通过 safe methods 在 `Send + Sync` 类型上暴露。VPN 核心运行在多线程 Tokio runtime 上,RPC、tunnel send/receive 和统计快照可能并发访问这些 counters;这会造成 Rust 层面的数据竞争和未定义行为,不只是“统计不准”。完成后应保证所有共享计数使用 atomic 或 lock-backed primitive,且新增并发测试证明 safe API 可多线程调用。
|
||||
|
||||
## 当前状态
|
||||
|
||||
- `easytier/src/common/stats_manager.rs` — 通用 metrics manager;当前 `UnsafeCounter` 和 `MetricData` 手写 `Send + Sync`。
|
||||
- `easytier/src/tunnel/stats.rs` — tunnel throughput 统计;当前单独实现一套 `UnsafeCell` counters。
|
||||
- `easytier/src/tunnel/filter.rs` — `StatsRecorderTunnelFilter` 在 send/receive filter 中更新 `Arc<Throughput>`。
|
||||
- `easytier/src/proto/rpc_impl/server.rs` — RPC server paths 会更新 stats manager counters,可作为并发使用背景参考,不要求修改。
|
||||
|
||||
当前代码摘录:
|
||||
|
||||
```rust
|
||||
// easytier/src/common/stats_manager.rs:406
|
||||
pub unsafe fn add(&self, delta: u64) {
|
||||
let ptr = self.value.get();
|
||||
unsafe {
|
||||
*ptr = (*ptr).saturating_add(delta);
|
||||
}
|
||||
}
|
||||
|
||||
// easytier/src/common/stats_manager.rs:455
|
||||
unsafe impl Send for UnsafeCounter {}
|
||||
unsafe impl Sync for UnsafeCounter {}
|
||||
|
||||
// easytier/src/common/stats_manager.rs:548
|
||||
pub fn add(&self, delta: u64) {
|
||||
unsafe {
|
||||
self.metric_data.counter.add(delta);
|
||||
self.metric_data.touch();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```rust
|
||||
// easytier/src/tunnel/stats.rs:64
|
||||
#[derive(Debug)]
|
||||
pub struct Throughput {
|
||||
tx_bytes: UnsafeCell<u64>,
|
||||
rx_bytes: UnsafeCell<u64>,
|
||||
tx_packets: UnsafeCell<u64>,
|
||||
rx_packets: UnsafeCell<u64>,
|
||||
}
|
||||
|
||||
// easytier/src/tunnel/stats.rs:83
|
||||
unsafe impl Send for Throughput {}
|
||||
unsafe impl Sync for Throughput {}
|
||||
```
|
||||
|
||||
```rust
|
||||
// easytier/src/tunnel/filter.rs:265
|
||||
fn before_send(&self, data: SinkItem) -> Option<SinkItem> {
|
||||
self.throughput.record_tx_bytes(data.buf_len() as u64);
|
||||
Some(data)
|
||||
}
|
||||
|
||||
// easytier/src/tunnel/filter.rs:270
|
||||
fn after_received(&self, data: StreamItem) -> Option<StreamItem> {
|
||||
match data {
|
||||
Ok(v) => {
|
||||
self.throughput.record_rx_bytes(v.buf_len() as u64);
|
||||
Some(Ok(v))
|
||||
}
|
||||
Err(e) => Some(Err(e)),
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
仓库约定:Rust 代码使用 `anyhow`/`thiserror` 做错误上下文,async tests 使用 `#[tokio::test]`;已有测试集中在 `easytier/src/tests/` 和各模块 `#[cfg(test)]` 中。保持现有 public method names,避免扩大 API 改动。
|
||||
|
||||
## 需要使用的命令
|
||||
|
||||
| Purpose | Command | Expected on success |
|
||||
|---------|---------|---------------------|
|
||||
| Format | `cargo fmt --all -- --check` | exit 0 |
|
||||
| Lint | `cargo clippy --all-targets --features full --all -- -D warnings` | exit 0, no warnings |
|
||||
| Feature check | `cargo hack check --package easytier --each-feature --exclude-features macos-ne --verbose` | exit 0 |
|
||||
| Targeted tests | `cargo test --package easytier stats_manager --features full -- --nocapture` | exit 0; new stats tests pass |
|
||||
| Targeted tests | `cargo test --package easytier tunnel::stats --features full -- --nocapture` | exit 0; new throughput tests pass |
|
||||
|
||||
## 临时目录约定
|
||||
|
||||
- 临时文件、scratch 目录和 disposable worktree 必须放在 `$HOME/tmp` 下。
|
||||
- 如果 `$HOME/tmp` 不存在且本计划需要临时空间,先创建它。
|
||||
- 不要把临时产物放进被修改仓库。
|
||||
|
||||
## 范围
|
||||
|
||||
**范围内**(只能修改这些文件):
|
||||
- `easytier/src/common/stats_manager.rs`
|
||||
- `easytier/src/tunnel/stats.rs`
|
||||
- `easytier/src/tunnel/filter.rs`(仅当 type/API 调整需要同步编译)
|
||||
- `easytier/src/tests/mod.rs` 或同文件内 `#[cfg(test)]` 测试(仅用于新增测试入口)
|
||||
|
||||
**范围外**(即使看起来相关也不要触碰):
|
||||
- `easytier/src/peers/*` route 或 RPC 行为;这些由后续计划处理。
|
||||
- `easytier-web/`、`easytier-gui/`、frontend packages。
|
||||
- 任何 public metric names、labels、serialized output shape 的语义变更。
|
||||
|
||||
## Git 工作流
|
||||
|
||||
- Branch: `advisor/001-thread-safe-metrics-throughput`
|
||||
- Commit message style follows existing conventional commits, for example `fix: clarify config parse errors` or `fix(connector): classify manual reconnect timeouts by stage`.
|
||||
- Do NOT push or open a PR unless the operator instructed it.
|
||||
|
||||
## 步骤
|
||||
|
||||
### 步骤 1:替换 `UnsafeCounter` 为 atomic-backed counter
|
||||
|
||||
在 `easytier/src/common/stats_manager.rs` 中将 `UnsafeCounter` 改为持有 `AtomicU64`。保留现有 `new`、`new_with_value`、`add`、`inc`、`get`、`reset`、`set` 方法名,但将它们改成 safe methods,使用 `Ordering::Relaxed` 即可,因为这些 counters 只做统计,不承载同步 happens-before 语义。
|
||||
|
||||
同时移除 `UnsafeCounter` 的 manual `unsafe impl Send/Sync`,让 compiler 从 `AtomicU64` 自动推导。
|
||||
|
||||
**验证**:`cargo test --package easytier stats_manager --features full -- --nocapture` → exit 0;如果此时没有匹配测试,命令应显示 0 failed。
|
||||
|
||||
### 步骤 2:处理 `MetricData::last_updated`
|
||||
|
||||
`MetricData` 当前持有 `UnsafeCell<Instant>`。不要继续共享可变 `Instant`。二选一:
|
||||
|
||||
- 推荐:将 last update 表示为 `AtomicU64`,存储从 `StatsManager` 创建时刻起的 monotonic micros 或 millis;读取时只在内部转换为需要的 age/duration。
|
||||
- 可接受:用 `parking_lot::Mutex<Instant>` 保护 `last_updated`,如果改动最小且性能足够。
|
||||
|
||||
选择方案后,移除 `MetricData` 的 manual `unsafe impl Send/Sync`。保持外部 behavior:counter update 后 last update 被刷新,过期清理逻辑仍能工作。
|
||||
|
||||
**验证**:`cargo clippy --all-targets --features full --all -- -D warnings` → exit 0, no warnings。
|
||||
|
||||
### 步骤 3:替换 `Throughput` 中的 `UnsafeCell` counters
|
||||
|
||||
在 `easytier/src/tunnel/stats.rs` 中将 `tx_bytes`、`rx_bytes`、`tx_packets`、`rx_packets` 改成 `AtomicU64`。`record_tx_bytes` 和 `record_rx_bytes` 使用 `fetch_add(..., Ordering::Relaxed)`;getter 使用 `load(Ordering::Relaxed)`。
|
||||
|
||||
更新 `Clone` 实现为加载旧值后创建新的 atomic counters。移除 `unsafe impl Send for Throughput` 和 `unsafe impl Sync for Throughput`。
|
||||
|
||||
**验证**:`cargo test --package easytier tunnel::stats --features full -- --nocapture` → exit 0;如果没有匹配测试,继续步骤 4 新增测试后重跑。
|
||||
|
||||
### 步骤 4:新增并发回归测试
|
||||
|
||||
为 `stats_manager` 添加一个多线程并发 increment 测试,建议放在 `easytier/src/common/stats_manager.rs` 的 `#[cfg(test)]` 模块中:创建一个 counter handle,启动多个 OS threads 或 `tokio::task::JoinSet`,每个 task 多次 `inc()`,最后断言总数等于预期。
|
||||
|
||||
为 `Throughput` 添加类似测试,创建 `Arc<Throughput>`,并发调用 `record_tx_bytes` 和 `record_rx_bytes`,最后断言 bytes 和 packets 全部精确匹配。
|
||||
|
||||
**验证**:`cargo test --package easytier stats_manager --features full -- --nocapture` 和 `cargo test --package easytier tunnel::stats --features full -- --nocapture` → exit 0;输出中新增测试通过。
|
||||
|
||||
### 步骤 5:运行完整相关门禁
|
||||
|
||||
运行格式、lint 和 feature check。
|
||||
|
||||
**验证**:
|
||||
- `cargo fmt --all -- --check` → exit 0。
|
||||
- `cargo clippy --all-targets --features full --all -- -D warnings` → exit 0。
|
||||
- `cargo hack check --package easytier --each-feature --exclude-features macos-ne --verbose` → exit 0。
|
||||
|
||||
## 测试计划
|
||||
|
||||
- 新增 `stats_manager` 并发 increment 测试:覆盖多线程 safe API 读写。
|
||||
- 新增 `Throughput` 并发 tx/rx 测试:覆盖 send/receive counters 同时更新。
|
||||
- 现有 tunnel filter 行为不需要改业务测试,只需保证编译和 clippy 通过。
|
||||
|
||||
## 完成标准
|
||||
|
||||
- [ ] `easytier/src/common/stats_manager.rs` 不再包含 `UnsafeCell`-backed counter 或 manual `unsafe impl Send/Sync` for metric data。
|
||||
- [ ] `easytier/src/tunnel/stats.rs` 不再包含 `UnsafeCell<u64>` 或 manual `unsafe impl Send/Sync` for `Throughput`。
|
||||
- [ ] 新增并发测试存在并通过。
|
||||
- [ ] `cargo fmt --all -- --check` exits 0。
|
||||
- [ ] `cargo clippy --all-targets --features full --all -- -D warnings` exits 0。
|
||||
- [ ] `cargo hack check --package easytier --each-feature --exclude-features macos-ne --verbose` exits 0。
|
||||
- [ ] 没有修改范围外文件(`git status --short` 仅显示本计划范围内文件和 `plans/README.md` 状态更新)。
|
||||
- [ ] 已更新 `plans/README.md` 中本计划的状态行。
|
||||
|
||||
## STOP 条件
|
||||
|
||||
- 当前状态中列出位置的代码与摘录不匹配。
|
||||
- 你发现 `last_updated` 的 public API 依赖真实 `Instant` 值,无法用 atomic duration 或 mutex 在范围内保持行为。
|
||||
- 修复需要改变 metrics output schema、metric names 或 label semantics。
|
||||
- `cargo clippy` 因 atomic ordering 或 dead code 问题连续两次失败且无法在范围内解决。
|
||||
|
||||
## 维护说明
|
||||
|
||||
- 未来新增统计 primitive 时禁止再用 `UnsafeCell` + manual `Send/Sync` 暴露 safe shared mutation;默认使用 atomics 或明确锁。
|
||||
- reviewer 应重点检查 atomic ordering 是否足够、是否移除了所有 unsafe shared counter paths、测试是否真的并发执行。
|
||||
- 本计划不优化 metrics aggregation 性能;只消除 UB 和数据竞争风险。
|
||||
@@ -0,0 +1,160 @@
|
||||
# 计划 002:为 peer RPC/control packet 队列加入背压和过载行为
|
||||
|
||||
> **执行者说明**:按步骤执行本计划。每一步都必须运行验证命令,并确认结果符合预期后再继续。如果触发“STOP 条件”中的任一情况,立即停止并报告,不要自行发挥。完成后更新 `plans/README.md` 中本计划的状态行,除非 reviewer 明确说明由他们维护索引。
|
||||
>
|
||||
> **漂移检查(首先运行)**:`git diff --stat 78146d16..HEAD -- easytier/src/peers/peer_manager.rs easytier/src/peers/foreign_network_manager.rs easytier/src/common/stats_manager.rs easytier/src/tests`
|
||||
> 如果本计划写成后任何范围内文件发生变化,继续前必须对照“当前状态”中的摘录与实时代码;如果不匹配,按 STOP 条件处理。
|
||||
|
||||
## 状态
|
||||
|
||||
- **优先级**: P1
|
||||
- **工作量**: M
|
||||
- **风险**: MED
|
||||
- **依赖**: plans/001-thread-safe-metrics-throughput.md
|
||||
- **类别**: perf
|
||||
- **计划生成于**: commit `78146d16`, 2026-06-18
|
||||
|
||||
## 为什么重要
|
||||
|
||||
Peer RPC/control packet transport 当前使用 `mpsc::unbounded_channel()`,network-facing packet processor 对每个 RPC packet 直接 `send(...).unwrap()`。如果远端或本地 relay 突发控制面 packet,队列可以无限增长,导致内存膨胀和控制面延迟;如果 receiver 关闭,`unwrap()` 还会 panic。完成后应有明确 bounded capacity、drop/backpressure policy 和可观测 drop 计数。
|
||||
|
||||
## 当前状态
|
||||
|
||||
- `easytier/src/peers/peer_manager.rs` — local peer RPC transport 队列和 packet processor。
|
||||
- `easytier/src/peers/foreign_network_manager.rs` — foreign-network RPC transport 队列和 relay/local packet ingestion。
|
||||
- `easytier/src/common/stats_manager.rs` — 如果 001 已完成,应复用线程安全 metrics 记录 queue drops。
|
||||
|
||||
当前代码摘录:
|
||||
|
||||
```rust
|
||||
// easytier/src/peers/peer_manager.rs:275
|
||||
// TODO: remove these because we have impl pipeline processor.
|
||||
let (peer_rpc_tspt_sender, peer_rpc_tspt_recv) = mpsc::unbounded_channel();
|
||||
```
|
||||
|
||||
```rust
|
||||
// easytier/src/peers/peer_manager.rs:1245
|
||||
struct PeerRpcPacketProcessor {
|
||||
peer_rpc_tspt_sender: UnboundedSender<ZCPacket>,
|
||||
}
|
||||
|
||||
// easytier/src/peers/peer_manager.rs:1257
|
||||
self.peer_rpc_tspt_sender.send(packet).unwrap();
|
||||
```
|
||||
|
||||
```rust
|
||||
// easytier/src/peers/foreign_network_manager.rs:362
|
||||
let (rpc_transport_sender, peer_rpc_tspt_recv) = mpsc::unbounded_channel();
|
||||
|
||||
// easytier/src/peers/foreign_network_manager.rs:529
|
||||
rpc_sender.send(zc_packet).unwrap();
|
||||
```
|
||||
|
||||
仓库约定:control-plane errors 通常通过 `tracing::{debug,warn,error}` 记录;packet hot path 应避免 blocking await。已有 data-plane queues elsewhere 倾向显式容量和丢弃策略;本计划应保持 hot path 非阻塞。
|
||||
|
||||
## 需要使用的命令
|
||||
|
||||
| Purpose | Command | Expected on success |
|
||||
|---------|---------|---------------------|
|
||||
| Format | `cargo fmt --all -- --check` | exit 0 |
|
||||
| Lint | `cargo clippy --all-targets --features full --all -- -D warnings` | exit 0, no warnings |
|
||||
| Feature check | `cargo hack check --package easytier --each-feature --exclude-features macos-ne --verbose` | exit 0 |
|
||||
| Targeted tests | `cargo test --package easytier peer_manager --features full -- --nocapture` | exit 0; new queue tests pass if present |
|
||||
|
||||
## 临时目录约定
|
||||
|
||||
- 临时文件、scratch 目录和 disposable worktree 必须放在 `$HOME/tmp` 下。
|
||||
- 如果 `$HOME/tmp` 不存在且本计划需要临时空间,先创建它。
|
||||
- 不要把临时产物放进被修改仓库。
|
||||
|
||||
## 范围
|
||||
|
||||
**范围内**(只能修改这些文件):
|
||||
- `easytier/src/peers/peer_manager.rs`
|
||||
- `easytier/src/peers/foreign_network_manager.rs`
|
||||
- `easytier/src/common/stats_manager.rs`(仅用于添加/复用 drop metric names;不要重做 001)
|
||||
- `easytier/src/tests/*`(仅新增/调整本计划相关测试)
|
||||
|
||||
**范围外**(即使看起来相关也不要触碰):
|
||||
- RPC protocol message definitions and generated protobuf code。
|
||||
- Routing semantics、credential trust、foreign network topology logic。
|
||||
- Frontend, web server, GUI。
|
||||
|
||||
## Git 工作流
|
||||
|
||||
- Branch: `advisor/002-bound-peer-rpc-queues`
|
||||
- Commit message style follows existing conventional commits, for example `fix: route_update message is not lag`.
|
||||
- Do NOT push or open a PR unless the operator instructed it.
|
||||
|
||||
## 步骤
|
||||
|
||||
### 步骤 1:定义 bounded capacity 和 overload policy
|
||||
|
||||
在两个文件中引入同一个小常量,建议名称为 `PEER_RPC_PACKET_QUEUE_CAPACITY`,初始值建议 `1024` 或 `4096`。如果已有相近 queue capacity 常量,复用仓库风格。
|
||||
|
||||
Policy 必须明确:packet hot path 不等待;当队列满或 receiver closed 时,丢弃当前 RPC/control packet,记录 `tracing::warn!` 或 rate-limited debug,并增加 drop counter。不要 panic。
|
||||
|
||||
**验证**:`cargo fmt --all -- --check` → exit 0。
|
||||
|
||||
### 步骤 2:替换 `peer_manager.rs` 的 unbounded channel
|
||||
|
||||
将 `mpsc::unbounded_channel()` 替换为 `mpsc::channel(PEER_RPC_PACKET_QUEUE_CAPACITY)`。更新 `RpcTransport`、`PeerRpcPacketProcessor` 字段类型,从 `UnboundedSender`/unbounded receiver 改成 bounded `Sender`/`Receiver`。
|
||||
|
||||
在 `try_process_packet_from_peer` 中不要 `.await`,使用 `try_send(packet)`。如果 `Full` 或 `Closed`,记录并返回 `None`,保持原有“这是 RPC packet,不再进入 data-plane pipeline”的行为。
|
||||
|
||||
**验证**:`cargo test --package easytier peer_manager --features full -- --nocapture` → exit 0;如果没有匹配测试,至少必须编译通过。
|
||||
|
||||
### 步骤 3:替换 `foreign_network_manager.rs` 的 unbounded channel
|
||||
|
||||
同样将 foreign-network RPC transport 改为 bounded channel,并在 ingestion path 使用 `try_send(zc_packet)`。不得保留 `unwrap()`。
|
||||
|
||||
如果两个文件都需要相同 helper,优先在各文件内保持小函数,避免为了复用引入新模块。最小正确改动优先。
|
||||
|
||||
**验证**:`cargo test --package easytier foreign_network_manager --features full -- --nocapture` → exit 0;如果没有匹配测试,至少必须编译通过。
|
||||
|
||||
### 步骤 4:添加队列满/receiver closed 的单元测试或小型回归测试
|
||||
|
||||
尽量在模块内新增不依赖真实网络 namespace 的测试:创建 bounded channel 容量为 1,填满后调用封装的 send helper,断言不会 panic 且返回/drop counter 行为正确。如果代码结构不允许直接测试 private helper,可以抽出一个 file-local helper function,例如 `try_enqueue_rpc_packet(...) -> bool`,测试 helper。
|
||||
|
||||
不要为了测试启动完整三节点网络;这属于慢集成测试,不适合验证 queue behavior。
|
||||
|
||||
**验证**:`cargo test --package easytier peer_rpc_queue --features full -- --nocapture` → exit 0;如果测试名不同,使用实际新增测试过滤器,输出中新增测试通过。
|
||||
|
||||
### 步骤 5:运行完整相关门禁
|
||||
|
||||
**验证**:
|
||||
- `cargo fmt --all -- --check` → exit 0。
|
||||
- `cargo clippy --all-targets --features full --all -- -D warnings` → exit 0。
|
||||
- `cargo hack check --package easytier --each-feature --exclude-features macos-ne --verbose` → exit 0。
|
||||
|
||||
## 测试计划
|
||||
|
||||
- 新增 queue helper tests,覆盖队列未满、队列满、receiver closed 三种情况。
|
||||
- 如果添加 drop metric,测试满队列时 counter 增加。
|
||||
- 不要求新增 full network integration test;bounded queue behavior 应在 unit-level 可验证。
|
||||
|
||||
## 完成标准
|
||||
|
||||
- [ ] `peer_manager.rs` 不再为 peer RPC transport 使用 `mpsc::unbounded_channel()`。
|
||||
- [ ] `foreign_network_manager.rs` 不再为 foreign-network RPC transport 使用 `mpsc::unbounded_channel()`。
|
||||
- [ ] 相关 packet enqueue path 不再调用 `.unwrap()`。
|
||||
- [ ] 满队列和 receiver closed 有明确非 panic 行为。
|
||||
- [ ] 新增或更新测试覆盖 queue overload behavior。
|
||||
- [ ] `cargo fmt --all -- --check` exits 0。
|
||||
- [ ] `cargo clippy --all-targets --features full --all -- -D warnings` exits 0。
|
||||
- [ ] `cargo hack check --package easytier --each-feature --exclude-features macos-ne --verbose` exits 0。
|
||||
- [ ] 没有修改范围外文件。
|
||||
- [ ] 已更新 `plans/README.md` 中本计划的状态行。
|
||||
|
||||
## STOP 条件
|
||||
|
||||
- 001 尚未完成,而本计划需要新增 metrics/drop counters;此时先执行 001 或报告阻塞。
|
||||
- `PeerRpcManager` 或 transport trait 要求 unbounded receiver 类型且无法在范围内替换。
|
||||
- 正确实现需要改变 RPC protocol semantics 或 routing trust logic。
|
||||
- bounded queue 导致现有 integration tests 稳定失败,且不能通过容量或 policy 微调解决。
|
||||
|
||||
## 维护说明
|
||||
|
||||
- reviewer 应重点审查 drop policy 是否适合 control-plane:丢弃低优先级 sync packet 可以接受,但不能默默破坏必须可靠的 request/response path。
|
||||
- 后续如果出现 reconnect storm 或 route sync loss,应结合 drop metrics 调整 capacity。
|
||||
- 本计划不实现优先级队列;如果未来需要区分 `RpcReq`、`RpcResp`、`TaRpc` 优先级,应另写计划。
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user