From 23778a8f7c0352dcd33f3a151f7c3b3b93dc60b1 Mon Sep 17 00:00:00 2001 From: vibe-code Date: Wed, 12 Aug 2026 12:22:22 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20Snapshot-Offset=20=E9=99=90=E9=A2=9D?= =?UTF-8?q?=E7=AD=96=E7=95=A5=E6=9B=BF=E6=8D=A2=20BUFFER=20=E7=9B=B2?= =?UTF-8?q?=E7=8C=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 核心改动:不再用 BUFFER 猜滞后窗口消耗,而是用同步时点快照 直接计算累计限额。 - BUFFER=300000 → SMALL_BUFFER=50000(仅覆盖非消耗性跳降) - get_effective_limit: limit = B0 - SMALL_BUFFER,不依赖 current_usage - 同步循环:写 balance_at_sync + usage_at_sync 替代 remaining/stable_limit - 突发检测/低额度判断:读 balance_at_sync 替代 remaining - volc_consume.py: vls.BUFFER → vls.SMALL_BUFFER - debug/debug_volc_sync.py: 同步更新 - docs/volc-quota-notes.md: 新增 §7.6 v6 设计文档,更新各节口径 --- debug/debug_volc_sync.py | 6 +- docs/volc-quota-notes.md | 142 +++++++++++++++++++++--------- volc_consume.py | 6 +- volc_limit_sync.py | 186 ++++++++++++++------------------------- 4 files changed, 169 insertions(+), 171 deletions(-) diff --git a/debug/debug_volc_sync.py b/debug/debug_volc_sync.py index 93488dd..4b144f6 100644 --- a/debug/debug_volc_sync.py +++ b/debug/debug_volc_sync.py @@ -20,7 +20,7 @@ SECRET_KEY = os.getenv( ) HOST = "open.volcengineapi.com" REGION = "cn-beijing" -BUFFER = 300000 +SMALL_BUFFER = 50000 # Suffixes to clean from Volcengine response SUFFIXES_TO_REMOVE = [ @@ -203,10 +203,10 @@ if __name__ == "__main__": print("\n--- Final Model Balances ---") for model, balance in model_balances.items(): - remaining = balance - BUFFER + remaining = balance - SMALL_BUFFER print(f"Model: {model}") print(f" Balance : {balance:,}") - print(f" Buffer : {BUFFER:,}") + print(f" SmallBuffer: {SMALL_BUFFER:,}") print(f" Remaining: {remaining:,}") # Simulate Redis Keys diff --git a/docs/volc-quota-notes.md b/docs/volc-quota-notes.md index 17647f8..f7ef43c 100644 --- a/docs/volc-quota-notes.md +++ b/docs/volc-quota-notes.md @@ -7,9 +7,8 @@ > - 2026-08-12 对话澄清（扣费口径、余额 API 滞后、限额方案演进） > - 2026-08-12 生产日志观察（deepseek-v4-flash-ga-260731） > -> 代码现状：v5 = 恢复 v2 骨架（`stable_limit = usage + balance − BUFFER`），2026-08-12 修正； -> v4（balance−BUFFER 冻结快照）当天实施后即发现误拦已回退，见 §7.3/§7.5。 -> 工作区在 HEAD `c6c1224` 之上含 v5 修正。 +> 代码现状:v6 = Snapshot-Offset(`limit = balance_at_sync + usage_at_sync − SMALL_BUFFER`), +> 2026-08-12 修改;BUFFER 盲猜法已废弃,见 §7.6。 --- @@ -89,9 +88,9 @@ - 观察结论：AvailableAmount 不随今日消耗实时递减（至少滞后 20 万级，上限未知）。 - **后果**：一切「余额随消耗实时递减」的公式（v2 的 `usage + balance` 池恒定假设）在 滞后下失效——usage 涨而 balance 不动 → 池膨胀 → 限额虚高 → 放行到扣费区。 -- 对策方向：以**最近同步快照**的余额为当日免费池上限（本地 usage 实时累计撞线）， - 快照频率由同步间隔（30min 常规 + 低余额 5/10min + 突发阈值触发）保障； - 滞后窗口内的消耗用 BUFFER 覆盖（见 §7.3）。 +- 最终对策：Snapshot-Offset（§7.6），不再依赖 `usage + balance` 守恒，而是用同步时点 + 的快照 `balance_at_sync + usage_at_sync = B0` 直接计算累计限额，两次同步间常数， + 实时 usage 自然抵消滞后窗口内的消耗。 --- @@ -106,19 +105,18 @@ Set Redis limit:dynamic:volcengine:remaining:deepseek-v4-flash-ga-260731 = 1,258 📊 [Usage] deepseek-v4-flash-ga-260731:ba0383d3: +38083 (Today: 1,279,904 | Rem: 1,558,307 | Limit: 2,000,000) ``` -字段口径： +字段口径: | 字段 | 含义 | 计算 | |---|---|---| -| Bal | 火山 API 余额（ListResourcePackages AvailableAmount 之和） | 同步快照 | -| Rem（Checking 行） | 入 Redis 的剩余 | Bal − BUFFER | -| Rem（Usage 行） | 展示用余额 | Redis remaining + BUFFER = Bal（展示与 API 一致） | +| Bal | 火山 API 余额(ListResourcePackages AvailableAmount 之和) | 同步快照 | +| BalAtSync | 同步时写入 Redis 的余额快照 | = Bal | +| UsageAtSync | 同步时写入 Redis 的累计消耗 | Redis `daily_usage` 求和 | | Today | 本地今日采集量 | Redis `daily_usage` 逐请求累加 | -| Limit | 生效限额 | min(2M, stable_limit) | -| StableLimit（v2） | 动态限额 | usage + balance − BUFFER（随 usage 膨胀，见 §7.1） | +| Limit | 生效限额 | min(2M, balance_at_sync + usage_at_sync − SMALL_BUFFER) | -**Today > Rem 的原因**：Today 是单日流量，Rem 是存量余额快照，两套口径本就不该比较。 -火山网页「今日采集量 1,696,135/2,000,000」对应 Today 与 2M 配额，与余额无关。 +**Today > BalAtSync 的原因**:Today 是单日流量,BalAtSync 是存量余额快照,两套口径本就不该比较。 +火山网页「今日采集量 1,696,135/2,000,000」对应 Today 与 2M 配额,与余额无关。 **扣费结论（2026-08-12 15:09 实测修正）**：14:59:51 同步 U=1,696,135 + B=596,869， 守恒反推今日初始免费池 B0 = 2,293,004 → 今日消耗 1,696,135 < B0 → **当日无扣费**。 @@ -134,14 +132,14 @@ Set Redis limit:dynamic:volcengine:remaining:deepseek-v4-flash-ga-260731 = 1,258 | key | 读写方 | 说明 | |---|---|---| | `daily_usage:{北京日期}:{model}:{hash}` | LiteLLM 成功回调 INCR + 消耗任务 INCR | 当日采集量，TTL 48h，北京 00:00 滚动 | -| `limit:dynamic:volcengine:remaining:{model}` | 守护线程写 | 余额 − BUFFER（同步频率驱动、展示） | -| `limit:dynamic:volcengine:stable_limit:{model}` | 守护线程写 | 稳定限额（v5：usage+balance−BUFFER） | +| `limit:dynamic:volcengine:balance_at_sync:{model}` | 守护线程写 | 同步时点 API AvailableAmount 快照 | +| `limit:dynamic:volcengine:usage_at_sync:{model}` | 守护线程写 | 同步时点 Redis 累计消耗快照 | | `consume:last_run:{日期}` | 消耗任务 | 23:00 当日去重，TTL 48h | ### 6.2 关键日志 - `🚫 [PRE-BLOCK] {model} daily limit reached ({usage}+{reserve}/{limit})`：请求前闸门 429 -- `📊 [Usage] {model}:{hash}: +{tokens} (Today: X | Rem: Bal | Limit: Y)`：成功记账 +- `📊 [Usage] {model}:{hash}: +{tokens} (Today: X | BalAtSync: Y | Limit: Z)`:成功记账 --- @@ -187,40 +185,96 @@ stable_limit = max(0, balance − BUFFER) # ❌ 错误：把「剩余余 > 30 万 BUFFER → 若滞后窗口内流量大，v2 会多放行到虚高 stable；可调大（如 50 万，env） - **部署**：v5 已改（工作区），需重启 litellm 容器加载；重启在生产主机由用户执行 -### 7.5 v5（2026-08-12 修正，当前）：恢复 v2 骨架 +### 7.5 v5(2026-08-12 修正,已废弃):恢复 v2 骨架 ``` stable_limit = max(0, usage + balance − BUFFER) ``` -- 与 v2（§7.1）公式相同，区别在**论证与认识**：v2 数学正确（B 收敛后 = B0 − U）， - 不需要冻结快照；v4 的「剩余当上限」误拦由「usage + balance 桥接」天然避免 -- 实测验证（14:59:51 同步 + 15:09:51 生产日志）：U=1,696,135、B=596,869 → - stable=1,993,004 → 今日剩余可用 296,869（账上 59.7 万用到剩 30 万 BUFFER）✓ -- 防扣费语义不变：usage 撞 stable = 剩余恰为 BUFFER 时停，扣费结构性杜绝； - balance < BUFFER（U=0）→ stable=0 → 全拦 -- 保留：`get_effective_limit` 兜底链（stable → FALLBACK_LIMIT 90 万）、pre-block reserve、 - 软踢/恢复、低余额加密同步（600s/300s） +- 与 v2(§7.1)公式相同,区别在**论证与认识**:v2 数学正确(B 收敛后 = B0 − U), + 不需要冻结快照;v4 的「剩余当上限」误拦由「usage + balance 桥接」天然避免 +- 实测验证(14:59:51 同步 + 15:09:51 生产日志):U=1,696,135、B=596,869 → + stable=1,993,004 → 今日剩余可用 296,869(账上 59.7 万用到剩 30 万 BUFFER)✓ +- 防扣费语义不变:usage 撞 stable = 剩余恰为 BUFFER 时停,扣费结构性杜绝; + balance < BUFFER(U=0)→ stable=0 → 全拦 +- **已废弃原因**:BUFFER 盲猜滞后窗口消耗,高强度击穿低强度浪费。被 §7.6 Snapshot-Offset 替代。 + +### 7.6 v6(Snapshot-Offset,2026-08-12 当前):宝塔防扣费 + +``` +limit = balance_at_sync + usage_at_sync − SMALL_BUFFER(5万) + = B0 − SMALL_BUFFER +``` + +#### 核心思路 + +不再猜滞后消耗,而是利用一个事实:**同步时点的 balance + usage = 当日初始免费池 B0(B0 是常数,一天内不变)**。 + +**limit 是累计限额(从 0 到上限),不是剩余余额**。`_check_request_block` 用 `current_usage + reserve >= limit` 撞线,`current_usage` 自然抵消了滞后窗口内的增量消耗。 + +#### 为什么不需要 BUFFER + +旧方案用 BUFFER 猜「滞后窗口内消耗了多少」——猜不准。新方案直接记录同步时点的 `balance_at_sync` 和 `usage_at_sync`,闸门读实时 `current_usage` 去减,精确抵消: + +``` +# 旧:盲猜 +limit = usage + balance - BUFFER(300k) + +# 新:精确抵消 +limit = balance_at_sync + usage_at_sync - SMALL_BUFFER(5k) +``` + +**SMALL_BUFFER(5 万)只覆盖非消耗性余额跳降**(资源包过期、API 未结算残差),不是消耗量。 + +#### 不变量 + +**永远不扣费**:`usage + reserve >= B0 - SMALL_BUFFER` 撞线时,剩余实时余额 = `B0 - usage` = `B0 - (B0 - SMALL_BUFFER - reserve)` = `SMALL_BUFFER + reserve` > 0。扣费在余额归零时才发生,结构性杜绝。 + +#### 执行流程 + +**同步线程**(每 5~30min / 突发触发): +1. 拉 API → 取到 balance +2. 读 Redis 取 current_usage +3. 写入 `balance_at_sync` = balance, `usage_at_sync` = current_usage (TTL 24h) +4. 不再写入 `remaining`/`stable_limit` 旧键 + +**请求闸门**: +1. 读 `balance_at_sync` + `usage_at_sync` +2. `limit = balance_at_sync + usage_at_sync - SMALL_BUFFER` +3. `usage + reserve >= limit` → 429 + +#### 与旧方案对比 + +| 场景 | 旧(BUFFER 30万) | 新(Snapshot-Offset) | +|---|---|---| +| 高强度消耗 | 滞后窗口内消耗 > 30 万 → 击穿扣费 | 实时 usage 撞 B0 线,不放行 | +| 低强度消耗 | 浪费 30 万额度(实际只用了 5 万) | 只留 5 万 SMALL_BUFFER | +| 新包到账 | 下次同步 stable limit 跳升 | 下次同步 B0 跳升,一致 | +| 包过期 | 余额跳降,limit 偏高至下次同步 | 同左,但 SMALL_BUFFER=5 万风险窗口小 | + +#### 遗留风险 + +1. **同步时点 B0 本身滞后**:API 的 AvailableAmount 在同步时点就已经滞后了 L0。这个量无法通过抵消消除。唯一缓解:同步频率(低余额 5min) + 突发检测。 +2. **多包同时过期 > SMALL_BUFFER**:概率低,无法绝对防御。可调大 SMALL_BUFFER 或监控包过期时间。 --- -## 8. 排障速查（今日结论汇总） +## 8. 排障速查(今日结论汇总) -1. 看到 Today > Rem 不是异常，先分清口径（§5）；判扣费用「守恒初始池」（U+B 反推）， - 不要用任意时点快照（13:36 滞后快照曾误判「超余额扣费」，实际无扣费） -2. 余额显示不动 ≠ 余额充足，是 API 滞后；同步快照 + BUFFER 兜滞后窗口 -3. 判断「还能免费采多少」：`(usage + balance − BUFFER) − 今日已采`（v5 已落地） -4. 消耗任务 23:00 直连 Ark 绕过 LiteLLM 闸门（防自杀），其自身 clamp：`plan ≤ 2M − usage`、 - `plan ≤ 余额快照 − 已烧 − BUFFER`，停止条件三闭环 -5. 余额滞后收敛后守恒成立（B = B0 − U）；30 天过期蒸发是余额下跌的另一来源， - 与消耗扣减区分（如 08-12 余额 1,558,307 → 596,869，其中约 50 万为滞后修正 + 过期） +1. 看到 Today > BalAtSync 不是异常,先分清口径(§5);判扣费用「守恒初始池」(BalAtSync+UsageAtSync 反推), + 不要用任意时点快照(13:36 滞后快照曾误判「超余额扣费」,实际无扣费) +2. 余额显示不动 ≠ 余额充足,是 API 滞后;Snapshot-Offset 不再依赖 BUFFER 覆盖,而是直接用 + `balance_at_sync + usage_at_sync - current_usage` 实时抵消 +3. 判断「还能免费采多少」:`balance_at_sync + usage_at_sync - SMALL_BUFFER - 今日已采`(v6) +4. 消耗任务 23:00 直连 Ark 绕过 LiteLLM 闸门(防自杀),其自身 clamp:`plan ≤ 2M − usage`、 + `plan ≤ 余额快照 − SMALL_BUFFER`,停止条件三闭环 +5. 余额滞后不改变 B0 守恒;30 天过期蒸发是余额下跌的另一来源, + 与消耗扣减区分(如 08-12 余额 1,558,307 → 596,869,其中约 50 万为滞后修正 + 过期)## 9. 待确认/风险清单 ---- - -## 9. 待确认/风险清单 - -- [ ] 余额 API 滞后上限实测（决定 BUFFER 是否调大，现 30 万；13:36 实测高估 ~50 万） -- [x] v4 误拦发现与回退（2026-08-12 15:09 生产日志实证，§7.3） -- [x] v5 恢复 v2 骨架（2026-08-12，§7.5；待生产主机重启生效） -- [ ] 消耗任务口径是否同步收紧（可选：其停止条件与闸门同一把尺子已满足） -- [ ] 空窗期（0:00–返还到账）行为观察：v5 下余额红线自动收紧，确认无异常误杀 +- [ ] 余额 API 滞后上限实测(决定是否需要进一步压缩 SMALL_BUFFER 或保留 5 万) +- [x] v4 误拦发现与回退(2026-08-12 15:09 生产日志实证,§7.3) +- [x] v5 废弃(2026-08-12,§7.5;被 v6 Snapshot-Offset 替代) +- [x] v6 实施(2026-08-12,§7.6;待生产主机重启生效) +- [ ] 消耗任务口径同步收紧(已改为 vls.SMALL_BUFFER,确认与闸门一致) +- [ ] 空窗期(0:00–返还到账)行为观察:v6 下 B0 自动收紧,确认无异常误杀 +- [ ] 数据库泄漏:512 亿 token 消耗记录(08-12 一天 deepseek-v4 烧了 512 亿)导致 Redis 内存膨胀;触发监控/清理机制 \ No newline at end of file diff --git a/volc_consume.py b/volc_consume.py index afdafa0..32c3e2f 100644 --- a/volc_consume.py +++ b/volc_consume.py @@ -429,7 +429,7 @@ def run_consume_model(model, source_groups, order, packages=None, dry_run=False) 对单个监控模型执行"洗过期包"消耗。 目标 = Σ窗口内包计划（均分×1.1/冲刺），clamp： - 采集上限：plan ≤ CONSUME_DAILY_CAP − 当日已消耗 - - 防扣费：plan ≤ 余额快照 − BUFFER + - 防扣费:plan ≤ 余额快照 − SMALL_BUFFER 停止条件（每轮检查）：daily_usage ≥ 2M / 余额快照−已烧 ≤ 0 / 已烧 ≥ plan :param packages: 复用 run_consume_all 的一次拉包结果；None 时自行拉取 """ @@ -451,8 +451,8 @@ def run_consume_model(model, source_groups, order, packages=None, dry_run=False) usage_now = _current_daily_usage(model) # clamp 1: 采集上限（超出部分不返还，白烧） plan = min(plan, CONSUME_DAILY_CAP - usage_now) - # clamp 2: 余额快照 − BUFFER（脚本自身不烧穿余额，防扣费） - plan = min(plan, max(0, balance - vls.BUFFER)) +# clamp 2: 余额快照 − SMALL_BUFFER(脚本自身不烧穿余额,防扣费) + plan = min(plan, max(0, balance - vls.SMALL_BUFFER)) plan = int(max(0, plan)) if plan < CONSUME_MIN_PLAN: diff --git a/volc_limit_sync.py b/volc_limit_sync.py index 1527a28..709fbe1 100644 --- a/volc_limit_sync.py +++ b/volc_limit_sync.py @@ -93,7 +93,7 @@ HOST = "open.volcengineapi.com" REGION = "cn-beijing" # Daemon Config -BUFFER = 300000 # User requested 30w buffer +SMALL_BUFFER = 50000 # 覆盖资源包过期/API 未结算等非消耗性余额跳降 CHECK_INTERVAL = 10 # 前置闸门预留量：请求开始判定 usage + reserve >= limit 即 429。 @@ -161,7 +161,7 @@ MONITORED_MODEL_ALIASES = {} HARD_DAILY_LIMITS = { # 火山免费额度规则（2026-08）：每模型每日采集上限 2M，次日按采集量发放资源包。 # 请求前闸门（async_pre_call_hook）已不滞后 → 无需打折（旧 1.8M/480 万为 post-hoc 时代遗产）。 - # 统一 2M，余额相关的动态限额由 stable_limit（usage+balance−BUFFER）表达。 + # 统一 2M，余额相关的动态限额由 balance_at_sync - (usage_now - usage_at_sync) 实时。 "__DEFAULT__": 2_000_000, } @@ -407,24 +407,26 @@ def calculate_effective_limit(static_limit, dynamic_limit): return float(static_limit) if static_limit is not None else None -def format_limit_debug_fields(static_limit=None, stable_limit=None, effective_limit=None): - """Render explicit limit labels for debug logs to avoid confusing stable and effective caps.""" +def format_limit_debug_fields(static_limit=None, balance_at_sync=None, usage_at_sync=None, effective_limit=None): + """Render explicit limit labels for debug logs.""" fields = [] - if static_limit is not None: fields.append(f"StaticLimit={int(float(static_limit)):,}") - if stable_limit is not None: - fields.append(f"StableLimit={int(float(stable_limit)):,}") + if balance_at_sync is not None: + fields.append(f"BalAtSync={int(float(balance_at_sync)):,}") + if usage_at_sync is not None: + fields.append(f"UsageAtSync={int(float(usage_at_sync)):,}") if effective_limit is not None: fields.append(f"EffectiveLimit={int(float(effective_limit)):,}") - return " | ".join(fields) if fields else "EffectiveLimit=unknown" def get_effective_limit(clean_model, api_base, current_usage=None): """ - Calculate effective limit using the Peak-Buffer strategy. - Target Limit = (Peak Daily Remaining) - Buffer. + Calculate effective limit using the Snapshot-Offset strategy. + limit = balance_at_sync + usage_at_sync - SMALL_BUFFER (= B0 - SMALL_BUFFER) + _check_request_block 用 current_usage + reserve >= limit 撞线,语义正确。 + current_usage 参数仅用于无同步数据的 fallback 路径,非公式必需。 """ if not redis_client: return get_static_limit(clean_model, api_base) @@ -432,34 +434,34 @@ def get_effective_limit(clean_model, api_base, current_usage=None): # 1. Get Static Limit (Hard Config) static_limit = get_static_limit(clean_model, api_base) - # 2. Get Dynamic Stable Limit (Calculated by Daemon) - # This limit is derived from: Peak_Daily_Balance - Buffer - # It is stable and immune to API latency spikes. - stable_limit_key = f"limit:dynamic:volcengine:stable_limit:{clean_model}" - stable_limit_val = redis_client.get(stable_limit_key) + # 2. Get Dynamic Limit from Snapshot + # balance_at_sync + usage_at_sync = 当日初始免费池 B0(常数) + # 累计限额 = B0 - SMALL_BUFFER,两次同步间不变 + balance_key = f"limit:dynamic:volcengine:balance_at_sync:{clean_model}" + usage_key = f"limit:dynamic:volcengine:usage_at_sync:{clean_model}" + + balance_val = redis_client.get(balance_key) + usage_val = redis_client.get(usage_key) effective_limit = None - if stable_limit_val is not None: + if balance_val is not None and usage_val is not None: try: - dynamic_limit = float(stable_limit_val) - effective_limit = calculate_effective_limit(static_limit, dynamic_limit) + balance_at_sync = float(balance_val) + usage_at_sync = float(usage_val) + # 累计限额 = B0 - SMALL_BUFFER,不依赖 current_usage + dynamic_limit = balance_at_sync + usage_at_sync - SMALL_BUFFER + dynamic_limit = max(0, dynamic_limit) + effective_limit = calculate_effective_limit(static_limit, dynamic_limit) except (ValueError, TypeError): effective_limit = calculate_effective_limit(static_limit, None) else: - # Fallback: 如果 daemon 尚未计算 stable limit（首次同步前/新增模型/TTL 过期）， - # 用保守兜底（FALLBACK_LIMIT），绝不裸奔到采集上限 —— 免费模型余额常态 < 2M， - # 裸奔 2M 会把余额不足的模型烧穿扣费。 + # Fallback: 无同步数据时用保守兜底 fallback = float(static_limit) if static_limit is not None else float(FALLBACK_LIMIT) fallback = min(fallback, float(FALLBACK_LIMIT)) effective_limit = calculate_effective_limit(fallback, None) - # Legacy fallback (Optional: Enable if you want immediate loose limits before first sync) - # dynamic_remaining = redis_client.get(f"limit:dynamic:volcengine:remaining:{clean_model}") - # if dynamic_remaining and current_usage: - # effective_limit = float(current_usage) + float(dynamic_remaining) - return effective_limit @@ -902,16 +904,14 @@ class DailyLimitHandler(CustomLogger): f" | Limit: {int(effective_limit):,}" if effective_limit is not None else "" ) - # Get Dynamic Remaining for Log - dynamic_key = f"limit:dynamic:volcengine:remaining:{clean_model}" - rem_val = redis_client.get(dynamic_key) - rem_display = "" - if rem_val is not None: + # Get Balance At Sync for Log + balance_key = f"limit:dynamic:volcengine:balance_at_sync:{clean_model}" + bal_val = redis_client.get(balance_key) + bal_display = "" + if bal_val is not None: try: - # Display Real Balance (Remaining + Buffer) - rem_int = int(float(rem_val)) - bal_est = rem_int + BUFFER - rem_display = f" | Rem: {bal_est:,}" + bal_int = int(float(bal_val)) + bal_display = f" | BalAtSync: {bal_int:,}" except Exception as e: print( f"DEBUG: Error calculating balance for {clean_model}: {e}" @@ -920,11 +920,11 @@ class DailyLimitHandler(CustomLogger): else: # Debug why key is missing if DEBUG_MODE: - print(f"DEBUG: Dynamic key not found: {dynamic_key}") + print(f"DEBUG: Balance key not found: {balance_key}") key_display = key.split(":", 2)[-1] log( - f"📊 [Usage] {key_display}: +{total_tokens} (Today: {current_total:,}{rem_display}{limit_str})", + f"📊 [Usage] {key_display}: +{total_tokens} (Today: {current_total:,}{bal_display}{limit_str})", level="INFO", ) @@ -1248,15 +1248,12 @@ def _maybe_trigger_daily_consume(): log(f"❌ 消耗任务调度失败: {e}", level="ERROR") -def compute_sync_interval(min_remaining, is_idle): - """按最小正剩余余额 + 空闲状态计算同步间隔（秒）。""" - # 余额分档退化：余额驱动的"慢速档"已无意义（日返还模型下余额高 = 常态， - # 返还/过期/消耗随时发生，慢同步只制造检测延迟窗口）。 - # 仅保留"低余额加密"：<10万 5min、<30万 10min（返还解锁/扣费检测要紧）； - # ≥30万 统一 30min，不再细分。真正的慢同步 = is_idle（base×4，上限 60min）。 - if min_remaining < 100_000: +def compute_sync_interval(min_balance, is_idle): + """按最小余额 + 空闲状态计算同步间隔(秒)。 + min_balance 是 balance_at_sync 中的最小值(实际余额,非扣 SMALL_BUFFER 后)。""" + if min_balance < 100_000: base_interval = 300 # 紧急 5 minutes - elif min_remaining < 300_000: + elif min_balance < 400_000: base_interval = 600 # 低余额 10 minutes else: base_interval = 1800 # 常规 30 minutes @@ -1304,7 +1301,7 @@ def active_monitor_loop(): "Starting Active Monitor Loop (Safety-Margin + Idle Awareness) / 正在启动主动监控循环 (安全余量 + 空闲感知)...", flush=True, ) - print(f"Buffer / 缓冲: {BUFFER}", flush=True) + print(f"SmallBuffer / 缓冲: {SMALL_BUFFER}", flush=True) while True: try: @@ -1334,11 +1331,11 @@ def active_monitor_loop(): # Calculate delta for this specific model m_delta = m_usage - last_usage_map.get(m, 0.0) - # Check if this model is considered "low quota" - rem_key = f"limit:dynamic:volcengine:remaining:{m}" - rem_val = redis_client.get(rem_key) - # 未知额度视为充足，避免首次启动时所有模型被误判为"低额度" - remaining = float(rem_val) if rem_val is not None else float(LOW_QUOTA_THRESHOLD) +# Check if this model is considered "low quota" + bal_key = f"limit:dynamic:volcengine:balance_at_sync:{m}" + bal_val = redis_client.get(bal_key) + # 未知额度视为充足,避免首次启动时所有模型被误判为"低额度" + remaining = float(bal_val) if bal_val is not None else float(LOW_QUOTA_THRESHOLD) if m_delta > 0 and remaining < LOW_QUOTA_THRESHOLD: any_low_quota_usage = True @@ -1377,9 +1374,9 @@ def active_monitor_loop(): model_delta = curr_usage - last_usage # 4. Get Current Balance for Threshold Calculation - rem_key = f"limit:dynamic:volcengine:remaining:{model}" - rem_val = redis_client.get(rem_key) - current_balance = float(rem_val) if rem_val else 0.0 + bal_key = f"limit:dynamic:volcengine:balance_at_sync:{model}" + bal_val = redis_client.get(bal_key) + current_balance = float(bal_val) if bal_val else 0.0 # 5. Determine Dynamic Threshold based on Balance # Rich models get loose limits, Poor models get strict limits. @@ -1427,19 +1424,16 @@ def active_monitor_loop(): configured_models_found = False for model_name, balance in model_balances.items(): - remaining = balance - BUFFER print( - f"DEBUG: Checking {model_name} (Bal: {int(balance):,}, Rem: {int(remaining):,}) (余额/剩余)", + f"DEBUG: Checking {model_name} (Bal: {int(balance):,})", flush=True, ) - # Target Resolution (Already handled in process_packages_breakdown mostly) - # But we'll treat model_name as the target since process_packages_breakdown now handles the list logic + # Target Resolution targets = [model_name] for target in targets: # STRICT FILTER: Only sync models explicitly configured with VolcEngine Base URL - # User Requirement: Do NOT sync or limit models that share names but use other providers (e.g. NewAPI) if target not in MONITORED_MODELS: if DEBUG_MODE: print( @@ -1448,16 +1442,14 @@ def active_monitor_loop(): ) continue - # Fetch Current Usage from Redis (to calculate Total Pool) - # We need to find the correct hash for this model from MONITORED_MODEL_PAIRS + # Fetch Current Usage from Redis current_usage = 0 target_hashes = [ p[1] for p in MONITORED_MODEL_PAIRS if p[0] == target ] - # Sum usage across all configured endpoints for this model (usually just one) + # Sum usage across all configured endpoints for this model for h in target_hashes: - # Replicate get_usage_key logic (Beijing Time UTC+8) now_ts = time.time() beijing_time_ts = now_ts + (8 * 3600) usage_date = time.strftime( @@ -1468,78 +1460,30 @@ def active_monitor_loop(): if val: current_usage += float(val) - # Use normalized key for storage - # Format: limit:dynamic:volcengine:remaining:{model_name} - dynamic_key = f"limit:dynamic:volcengine:remaining:{target}" + balance_key = f"limit:dynamic:volcengine:balance_at_sync:{target}" + usage_key = f"limit:dynamic:volcengine:usage_at_sync:{target}" - # ========================================== - # 📏 v5（恢复 v2 骨架）：stable_limit = max(0, usage + balance − BUFFER) - # ========================================== - # 论证（2026-08-12 实测修正）： - # 1. 「usage + balance = 当日初始免费池」数学正确的前提是 balance 随 - # 消耗实时递减 —— 实测（14:59:51 同步）U=1,696,135 + B=596,869 - # = 2,293,004 ≈ 守恒初始池，B 在滞后收敛后确实等于 B0 − U， - # 与理论剩余 597,004 几乎一致 → v2 公式本身是对的。 - # 2. v4 的 balance − BUFFER 是错误：把「剩余余额」当「从 0 累计的 - # 上限」，同步时 U>0 则 stable 低估初始池 → 有真实剩余额度却误拦 - # （15:09:51 实测：Bal=596,869 剩 59.7 万却被 usage≥296,869 全拦）。 - # 3. v2 残留风险仅剩「同步时点 B 滞后」（13:36 快照 1,558,307 高估 - # 真实剩余约 50 万，池虚高）→ 由 BUFFER + 低余额加密同步（600s） - # + burst 触发覆盖；滞后收敛后（1 小时内实测）池回到守恒值。 - # 4. 扣费判定 = 消耗 > 当下免费额度即扣费；今日消耗 1,696,135 < - # 初始池 2,293,004 → 无扣费（此前"超余额 13.8 万已扣费"基于滞后 - # 快照 1,558,307 的推断不成立）。 - # 结论：stable = usage + balance − BUFFER（v2 原式），BUFFER 覆盖 - # 同步窗口内的余额滞后。见 docs/volc-quota-notes.md §7.5。 - - stable_limit = int(max(0, current_usage + balance - BUFFER)) - - stable_limit_key = ( - f"limit:dynamic:volcengine:stable_limit:{target}" - ) - - # Store stable limit - redis_client.setex(stable_limit_key, 86400, int(stable_limit)) + # Store snapshot: balance_at_sync, usage_at_sync + redis_client.setex(balance_key, 86400, int(balance)) + redis_client.setex(usage_key, 86400, int(current_usage)) if DEBUG_MODE: static_limit = get_monitored_static_limit( target, target_hashes ) - effective_limit = calculate_effective_limit( - static_limit, stable_limit - ) print( - f"DEBUG: {target} Bal={int(balance):,} | Usage={int(current_usage):,} | {format_limit_debug_fields(static_limit=static_limit, stable_limit=stable_limit, effective_limit=effective_limit)}", + f"DEBUG: {target} Bal={int(balance):,} | Usage={int(current_usage):,} | BalAtSync={int(balance):,} UsageAtSync={int(current_usage):,}", flush=True, ) - if DEBUG_MODE: - print( - f"DEBUG: Checking {target} (Configured / 已配置) (Bal: {int(balance):,}, Rem: {int(remaining):,})", - flush=True, - ) - - # Set with expiry (24h) - redis_client.setex(dynamic_key, 86400, int(remaining)) - if DEBUG_MODE: - print( - f"DEBUG: Set Redis {dynamic_key} = {int(remaining):,}", - flush=True, - ) - - # 即时恢复：同步发现限额已恢复（usage < effective limit）时立刻解除软踢， - # 不等 restore_blocked_models 的 30min/4h 闸门 —— 否则日返还额度到账后 - # 模型仍被禁（有额度但 no healthy deployments），直到下一次恢复检查。 + # 即时恢复:同步发现限额已恢复(usage < effective limit)时立刻解除软踢, restore_model_if_healthy(target, target_hashes) - # Track min remaining for frequency control + # Track min balance for frequency control configured_models_found = True - # Only track POSITIVE remaining for frequency control - # If a model is 0 (dead), we shouldn't panic-poll the healthy ones. - # We rely on the Restore Loop to check dead models. - if remaining > 0 and remaining < temp_min_remaining: - temp_min_remaining = remaining + if balance > 0 and balance < temp_min_remaining: + temp_min_remaining = balance # If all models are dead (temp_min_remaining is still inf), set to 0 for logic but handle gracefully # If we found models but all were <= 0, then temp_min_remaining is inf.