Skip to content

Commit 5cb26f4

Browse files
committed
feat(ai-gateway): Phase1-4 优化 — 多Key池化/熔断器/Prompt防护/成本追踪/语义缓存/OTel
- 新增 config/key_pool.py: 多Key轮询池,突破单Key RPM限制 - 新增 providers/circuit_breaker.py: Provider级熔断器(CLOSED/OPEN/HALF_OPEN) - 新增 middleware/prompt_guard.py: Prompt注入防护(中英文正则+白名单) - 新增 cost/tracker.py + cost/budget.py: Token成本追踪+预算硬限制 - 新增 observability/metrics.py + tracing.py: Prometheus指标+OTel追踪 - 新增 cache/semantic_cache.py: n-gram Jaccard语义缓存(L2) - 新增 middleware/virtual_key.py: 虚拟Key签发(Phase4预实现) - 改造 middleware/auth.py: 生产环境缺JWT密钥拒绝启动 - 改造 config/fallback.py: 流式fallback添加总预算超时+熔断联动 - 改造 provider_bootstrap.py: Key池集成+后台健康探活(30s) - 改造 main.py: GZip压缩+并发Semaphore+新中间件注册 - 新增 docs/Foresight/ai-gateway-optimization-plan.md: 完整优化方案文档
1 parent ed2e656 commit 5cb26f4

16 files changed

Lines changed: 1973 additions & 41 deletions

File tree

‎docs/Foresight/ai-gateway-optimization-plan.md‎

Lines changed: 561 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
"""
2+
熵减 AI 网关 — 语义缓存(L2)
3+
4+
@ai-context: 在精确 hash 缓存(L1)之上,提供基于文本相似度的语义缓存(L2)。
5+
使用轻量级字符级 n-gram 指纹计算相似度(无需外部 embedding 模型依赖),
6+
相似度超过阈值时命中缓存。适用于 prompt 微变但语义相同的场景。
7+
@ai-context: 当前实现为内存 LRU(进程级),后续可升级为 Redis + RediSearch
8+
向量索引。设计为可选增强:未启用时不影响现有 L1 精确缓存。
9+
"""
10+
11+
import hashlib
12+
import logging
13+
import time
14+
from collections import OrderedDict
15+
from typing import Any
16+
17+
logger = logging.getLogger(__name__)
18+
19+
# 语义缓存配置
20+
SEMANTIC_CACHE_MAX_SIZE = 200 # 最大缓存条目数
21+
SEMANTIC_CACHE_TTL_SECONDS = 1800 # 默认 TTL(30 分钟)
22+
SIMILARITY_THRESHOLD = 0.85 # 相似度命中阈值
23+
24+
25+
def _ngram_fingerprint(text: str, n: int = 3) -> set[str]:
26+
"""计算文本的字符级 n-gram 指纹集合
27+
28+
用于快速相似度估算(Jaccard 系数)。
29+
比 embedding 轻量,适合网关侧低延迟场景。
30+
"""
31+
# 归一化:去空白、转小写
32+
normalized = "".join(text.lower().split())
33+
if len(normalized) < n:
34+
return {normalized}
35+
return {normalized[i:i + n] for i in range(len(normalized) - n + 1)}
36+
37+
38+
def jaccard_similarity(set_a: set[str], set_b: set[str]) -> float:
39+
"""计算两个集合的 Jaccard 相似度"""
40+
if not set_a or not set_b:
41+
return 0.0
42+
intersection = len(set_a & set_b)
43+
union = len(set_a | set_b)
44+
return intersection / union if union > 0 else 0.0
45+
46+
47+
class SemanticCacheEntry:
48+
"""语义缓存条目"""
49+
50+
__slots__ = ("fingerprint", "prompt_hash", "response", "created_at", "ttl", "feature")
51+
52+
def __init__(self, fingerprint: set[str], prompt_hash: str, response: dict, feature: str, ttl: int):
53+
self.fingerprint = fingerprint
54+
self.prompt_hash = prompt_hash
55+
self.response = response
56+
self.feature = feature
57+
self.created_at = time.time()
58+
self.ttl = ttl
59+
60+
@property
61+
def is_expired(self) -> bool:
62+
return (time.time() - self.created_at) > self.ttl
63+
64+
65+
class SemanticCache:
66+
"""语义缓存(L2)— 基于 n-gram Jaccard 相似度
67+
68+
特性:
69+
- LRU 淘汰 + TTL 过期
70+
- 按 feature 隔离(不同功能的缓存不互相命中)
71+
- 相似度阈值可配置
72+
- 线程安全(单进程 asyncio 环境下无需锁)
73+
"""
74+
75+
def __init__(
76+
self,
77+
max_size: int = SEMANTIC_CACHE_MAX_SIZE,
78+
default_ttl: int = SEMANTIC_CACHE_TTL_SECONDS,
79+
threshold: float = SIMILARITY_THRESHOLD,
80+
):
81+
self.max_size = max_size
82+
self.default_ttl = default_ttl
83+
self.threshold = threshold
84+
self._store: OrderedDict[str, SemanticCacheEntry] = OrderedDict()
85+
self._hits = 0
86+
self._misses = 0
87+
88+
def get(self, prompt: str, feature: str) -> dict[str, Any] | None:
89+
"""尝试语义匹配缓存
90+
91+
Args:
92+
prompt: 用户 prompt 文本
93+
feature: 功能标识(隔离不同功能)
94+
95+
Returns:
96+
缓存的响应 dict,未命中返回 None
97+
"""
98+
query_fp = _ngram_fingerprint(prompt)
99+
best_score = 0.0
100+
best_key: str | None = None
101+
102+
for key, entry in self._store.items():
103+
# 过期清理
104+
if entry.is_expired:
105+
del self._store[key]
106+
continue
107+
# 功能隔离
108+
if entry.feature != feature:
109+
continue
110+
# 相似度计算
111+
score = jaccard_similarity(query_fp, entry.fingerprint)
112+
if score > best_score:
113+
best_score = score
114+
best_key = key
115+
116+
if best_key and best_score >= self.threshold:
117+
# LRU:移到末尾
118+
self._store.move_to_end(best_key)
119+
self._hits += 1
120+
entry = self._store[best_key]
121+
logger.debug(
122+
"SemanticCache HIT: feature=%s, similarity=%.3f, age=%.0fs",
123+
feature, best_score, time.time() - entry.created_at,
124+
)
125+
return entry.response
126+
127+
self._misses += 1
128+
return None
129+
130+
def put(self, prompt: str, feature: str, response: dict, ttl: int | None = None) -> None:
131+
"""写入语义缓存
132+
133+
Args:
134+
prompt: 用户 prompt 文本
135+
feature: 功能标识
136+
response: AI 响应 dict
137+
ttl: 过期时间(秒),None 使用默认值
138+
"""
139+
prompt_hash = hashlib.sha256(prompt.encode()).hexdigest()[:16]
140+
fingerprint = _ngram_fingerprint(prompt)
141+
actual_ttl = ttl or self.default_ttl
142+
143+
# LRU 淘汰
144+
while len(self._store) >= self.max_size:
145+
self._store.popitem(last=False)
146+
147+
self._store[prompt_hash] = SemanticCacheEntry(
148+
fingerprint=fingerprint,
149+
prompt_hash=prompt_hash,
150+
response=response,
151+
feature=feature,
152+
ttl=actual_ttl,
153+
)
154+
155+
@property
156+
def hit_rate(self) -> float:
157+
"""缓存命中率"""
158+
total = self._hits + self._misses
159+
return self._hits / total if total > 0 else 0.0
160+
161+
@property
162+
def size(self) -> int:
163+
return len(self._store)
164+
165+
def get_stats(self) -> dict:
166+
"""获取缓存统计"""
167+
return {
168+
"size": self.size,
169+
"max_size": self.max_size,
170+
"hits": self._hits,
171+
"misses": self._misses,
172+
"hit_rate": round(self.hit_rate, 3),
173+
"threshold": self.threshold,
174+
}
175+
176+
def clear(self) -> None:
177+
"""清空缓存"""
178+
self._store.clear()
179+
self._hits = 0
180+
self._misses = 0
181+
182+
183+
# ============================================================
184+
# 全局单例
185+
# ============================================================
186+
187+
_semantic_cache: SemanticCache | None = None
188+
189+
190+
def get_semantic_cache() -> SemanticCache:
191+
"""获取全局语义缓存实例"""
192+
global _semantic_cache
193+
if _semantic_cache is None:
194+
_semantic_cache = SemanticCache()
195+
return _semantic_cache

‎server/ai-gateway/config/fallback.py‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ async def call_with_fallback_stream(
218218
219219
与 call_with_fallback_for_request 相同的 fallback 链逻辑,
220220
但 fn 返回 AsyncGenerator[str, None] 而非 dict。
221+
Phase1 加固:添加总预算超时保护(与非流式一致)+ 熔断器联动。
221222
222223
Args:
223224
app: FastAPI 应用实例
@@ -228,26 +229,48 @@ async def call_with_fallback_stream(
228229
Returns:
229230
tuple: (generator, provider_key, is_user_key)
230231
"""
232+
from providers.circuit_breaker import is_provider_available, get_circuit
233+
234+
budget = TIMEOUT_CONFIG.get(feature, 30) * 1.5
231235
user_api_key = getattr(request.state, "user_api_key", None)
232236

233237
if not user_api_key:
234-
# 无用户 Key,走服务端 fallback 链
238+
# 无用户 Key,走服务端 fallback 链(带总预算超时)
235239
chain = PROVIDER_FALLBACK_CHAIN.get(feature, ["fallback"])
240+
start_time = asyncio.get_event_loop().time()
236241

237242
for provider_key in chain:
243+
# 熔断器检查:跳过已熔断的 Provider
244+
if not is_provider_available(provider_key):
245+
logger.debug("Provider [%s] 已熔断,跳过 (stream feature=%s)", provider_key, feature)
246+
continue
247+
248+
# 剩余预算检查
249+
elapsed = asyncio.get_event_loop().time() - start_time
250+
if elapsed >= budget:
251+
logger.error("流式 fallback 预算耗尽: feature=%s, elapsed=%.1fs", feature, elapsed)
252+
break
253+
238254
provider = app.state.providers.get(provider_key)
239255
if not provider:
240256
continue
241257
model_name = _resolve_model_name(provider_key, feature)
242258
try:
243259
_FEATURE_CONTEXT.set(feature)
244260
gen = fn(provider, model_name)
261+
# 通知熔断器成功
262+
cb = get_circuit(provider_key)
263+
if cb:
264+
await cb.on_success()
245265
return gen, provider_key, False
246266
except Exception as e:
247267
logger.warning(
248268
"Provider [%s] stream failed for feature=%s: %s, trying next...",
249269
provider_key, feature, str(e),
250270
)
271+
cb = get_circuit(provider_key)
272+
if cb:
273+
await cb.on_failure()
251274
finally:
252275
_FEATURE_CONTEXT.set("")
253276

@@ -269,22 +292,37 @@ async def call_with_fallback_stream(
269292
finally:
270293
_FEATURE_CONTEXT.set("")
271294

272-
# 降级到服务端 fallback 链
295+
# 降级到服务端 fallback 链(带总预算超时)
273296
chain = PROVIDER_FALLBACK_CHAIN.get(feature, ["fallback"])
297+
start_time = asyncio.get_event_loop().time()
298+
274299
for provider_key in chain:
300+
if not is_provider_available(provider_key):
301+
continue
302+
303+
elapsed = asyncio.get_event_loop().time() - start_time
304+
if elapsed >= budget:
305+
break
306+
275307
provider = app.state.providers.get(provider_key)
276308
if not provider:
277309
continue
278310
model_name = _resolve_model_name(provider_key, feature)
279311
try:
280312
_FEATURE_CONTEXT.set(feature)
281313
gen = fn(provider, model_name)
314+
cb = get_circuit(provider_key)
315+
if cb:
316+
await cb.on_success()
282317
return gen, provider_key, False
283318
except Exception as e:
284319
logger.warning(
285320
"Provider [%s] stream failed for feature=%s: %s, trying next...",
286321
provider_key, feature, str(e),
287322
)
323+
cb = get_circuit(provider_key)
324+
if cb:
325+
await cb.on_failure()
288326
finally:
289327
_FEATURE_CONTEXT.set("")
290328

0 commit comments

Comments
 (0)