zqf
b1f93588f1
- 排除 .env / *.bak / 内部运维文档（README_INTERNAL.html） - config.py 默认密码已替换为占位符 CHANGE_ME_* - init_db.sql 移除生产数据库用户 GRANT 段 - README.html 数据库用户名已脱敏 - 保留：源码 + 公网 API 文档 + 建表 SQL（无授权语句）
291 lines
12 KiB
Python
291 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Memory System Validation Tests
|
|
|
|
Tests:
|
|
1. Create two teams (team_a, team_b)
|
|
2. Create agents in each team
|
|
3. Write personal memories with team-specific content
|
|
4. Semantic search with team isolation verification
|
|
5. Performance benchmarks (write latency, search latency, memory usage)
|
|
"""
|
|
import sys
|
|
import os
|
|
import time
|
|
import json
|
|
import statistics
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from config import Config
|
|
from memory_system import MemorySystem
|
|
|
|
# ANSI colors
|
|
GREEN = "\033[92m"
|
|
RED = "\033[91m"
|
|
YELLOW = "\033[93m"
|
|
CYAN = "\033[96m"
|
|
RESET = "\033[0m"
|
|
BOLD = "\033[1m"
|
|
|
|
def header(text):
|
|
print(f"\n{BOLD}{CYAN}{'='*60}{RESET}")
|
|
print(f"{BOLD}{CYAN}{text}{RESET}")
|
|
print(f"{BOLD}{CYAN}{'='*60}{RESET}")
|
|
|
|
def ok(msg):
|
|
print(f" {GREEN}✓{RESET} {msg}")
|
|
|
|
def fail(msg):
|
|
print(f" {RED}✗{RESET} {msg}")
|
|
|
|
def warn(msg):
|
|
print(f" {YELLOW}⚠{RESET} {msg}")
|
|
|
|
def measure(label, func, *args, **kwargs):
|
|
"""Run func and return (result, elapsed_ms)."""
|
|
start = time.perf_counter()
|
|
result = func(*args, **kwargs)
|
|
elapsed = (time.perf_counter() - start) * 1000
|
|
return result, elapsed
|
|
|
|
|
|
def main():
|
|
cfg = Config()
|
|
ms = MemorySystem(cfg)
|
|
|
|
# ── Health checks ────────────────────────────────────────────
|
|
header("Health Checks")
|
|
|
|
if ms.embedder.health_check():
|
|
ok("BGE embedding service is alive")
|
|
else:
|
|
fail("BGE embedding service not responding")
|
|
sys.exit(1)
|
|
|
|
if ms.redis.health_check():
|
|
ok("Redis connection OK")
|
|
else:
|
|
fail("Redis connection failed")
|
|
sys.exit(1)
|
|
|
|
ok("MySQL connection OK (implicit via queries)")
|
|
|
|
# ── Step 1: Create teams ─────────────────────────────────────
|
|
header("Step 1: Create Teams")
|
|
|
|
# Clean up first (idempotent)
|
|
try:
|
|
ms.delete_team("team_a")
|
|
ms.delete_team("team_b")
|
|
except:
|
|
pass
|
|
|
|
team_a, elapsed = measure("create_team", ms.create_team, "team_a", "Team Alpha", "First test team")
|
|
ok(f"Created team_a: {team_a['name']} ({elapsed:.1f}ms)")
|
|
|
|
team_b, elapsed = measure("create_team", ms.create_team, "team_b", "Team Beta", "Second test team")
|
|
ok(f"Created team_b: {team_b['name']} ({elapsed:.1f}ms)")
|
|
|
|
# ── Step 2: Create agents ────────────────────────────────────
|
|
header("Step 2: Create Agents")
|
|
|
|
agent_a, elapsed = measure("create_agent", ms.create_agent, "agent_alpha_1", "team_a", "Alpha Agent", "developer")
|
|
ok(f"Created agent_alpha_1 in team_a ({elapsed:.1f}ms)")
|
|
|
|
agent_b, elapsed = measure("create_agent", ms.create_agent, "agent_beta_1", "team_b", "Beta Agent", "developer")
|
|
ok(f"Created agent_beta_1 in team_b ({elapsed:.1f}ms)")
|
|
|
|
# ── Step 3: Write personal memories ──────────────────────────
|
|
header("Step 3: Write Personal Memories")
|
|
|
|
write_latencies = []
|
|
|
|
mem_a, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
|
"agent_alpha_1", "用户的编程偏好是 Python，喜欢用 Flask 和 FastAPI 框架",
|
|
importance=0.8, metadata={"topic": "preference"})
|
|
write_latencies.append(elapsed)
|
|
ok(f"team_a memory: '{mem_a['content'][:40]}...' ({elapsed:.1f}ms)")
|
|
|
|
mem_b, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
|
"agent_beta_1", "用户的编程偏好是 Java，喜欢用 Spring Boot 框架",
|
|
importance=0.8, metadata={"topic": "preference"})
|
|
write_latencies.append(elapsed)
|
|
ok(f"team_b memory: '{mem_b['content'][:40]}...' ({elapsed:.1f}ms)")
|
|
|
|
# Add more memories for richer testing
|
|
for i, content in enumerate([
|
|
"团队每周一开例会",
|
|
"项目截止日期是下个月底",
|
|
"用户不喜欢加班",
|
|
]):
|
|
_, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
|
"agent_alpha_1", content, importance=0.5 + i * 0.1)
|
|
write_latencies.append(elapsed)
|
|
|
|
for i, content in enumerate([
|
|
"团队使用 Jenkins 做 CI/CD",
|
|
"数据库用的是 PostgreSQL",
|
|
"代码审查需要至少两人通过",
|
|
]):
|
|
_, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
|
"agent_beta_1", content, importance=0.5 + i * 0.1)
|
|
write_latencies.append(elapsed)
|
|
|
|
ok(f"Wrote {len(write_latencies)} memories total")
|
|
|
|
# ── Step 4: Write team memories ──────────────────────────────
|
|
header("Step 4: Write Team Shared Memories")
|
|
|
|
tm_a, elapsed = measure("add_team_memory", ms.add_team_memory,
|
|
"team_a", "团队技术栈: Python, Flask, MySQL, Redis",
|
|
importance=0.9, category="tech_stack")
|
|
write_latencies.append(elapsed)
|
|
ok(f"team_a shared: '{tm_a['content'][:40]}...' ({elapsed:.1f}ms)")
|
|
|
|
tm_b, elapsed = measure("add_team_memory", ms.add_team_memory,
|
|
"team_b", "团队技术栈: Java, Spring Boot, PostgreSQL, Kafka",
|
|
importance=0.9, category="tech_stack")
|
|
write_latencies.append(elapsed)
|
|
ok(f"team_b shared: '{tm_b['content'][:40]}...' ({elapsed:.1f}ms)")
|
|
|
|
# ── Step 5: Semantic search with isolation ───────────────────
|
|
header("Step 5: Semantic Search + Team Isolation")
|
|
|
|
search_latencies = []
|
|
|
|
# Search team_a for "编程偏好"
|
|
results_a, elapsed = measure("search_personal_memories", ms.search_personal_memories,
|
|
"agent_alpha_1", "编程偏好", limit=5)
|
|
search_latencies.append(elapsed)
|
|
ok(f"team_a search '编程偏好' → {len(results_a)} results ({elapsed:.1f}ms)")
|
|
for r in results_a:
|
|
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
|
|
|
# Search team_b for "编程偏好"
|
|
results_b, elapsed = measure("search_personal_memories", ms.search_personal_memories,
|
|
"agent_beta_1", "编程偏好", limit=5)
|
|
search_latencies.append(elapsed)
|
|
ok(f"team_b search '编程偏好' → {len(results_b)} results ({elapsed:.1f}ms)")
|
|
for r in results_b:
|
|
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
|
|
|
# Verify isolation
|
|
header("Step 6: Cross-Team Isolation Verification")
|
|
|
|
a_contents = [r["content"] for r in results_a]
|
|
b_contents = [r["content"] for r in results_b]
|
|
|
|
python_in_a = any("Python" in c for c in a_contents)
|
|
java_in_a = any("Java" in c for c in a_contents)
|
|
python_in_b = any("Python" in c for c in b_contents)
|
|
java_in_b = any("Java" in c for c in b_contents)
|
|
|
|
if python_in_a and not java_in_a:
|
|
ok("team_a returns Python (not Java) ✓")
|
|
elif not a_contents:
|
|
warn("team_a search returned no results")
|
|
else:
|
|
fail(f"team_a isolation issue: found Java={java_in_a}, Python={python_in_a}")
|
|
|
|
if java_in_b and not python_in_b:
|
|
ok("team_b returns Java (not Python) ✓")
|
|
elif not b_contents:
|
|
warn("team_b search returned no results")
|
|
else:
|
|
fail(f"team_b isolation issue: found Python={python_in_b}, Java={java_in_b}")
|
|
|
|
# Also verify team shared memories
|
|
team_search_a, elapsed = measure("search_team_memories", ms.search_team_memories,
|
|
"team_a", "技术栈", limit=3)
|
|
search_latencies.append(elapsed)
|
|
ok(f"team_a shared search '技术栈' → {len(team_search_a)} results ({elapsed:.1f}ms)")
|
|
for r in team_search_a:
|
|
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
|
|
|
team_search_b, elapsed = measure("search_team_memories", ms.search_team_memories,
|
|
"team_b", "技术栈", limit=3)
|
|
search_latencies.append(elapsed)
|
|
ok(f"team_b shared search '技术栈' → {len(team_search_b)} results ({elapsed:.1f}ms)")
|
|
for r in team_search_b:
|
|
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
|
|
|
# ── Step 7: Working memory ──────────────────────────────────
|
|
header("Step 7: Working Memory (Redis)")
|
|
|
|
wm, elapsed = measure("add_working_memory", ms.add_working_memory,
|
|
"agent_alpha_1", "刚刚和用户讨论了部署方案")
|
|
ok(f"Added working memory ({elapsed:.1f}ms)")
|
|
|
|
wm_items, elapsed = measure("get_working_memories", ms.get_working_memories,
|
|
"agent_alpha_1", limit=10)
|
|
ok(f"Retrieved {len(wm_items)} working memories ({elapsed:.1f}ms)")
|
|
|
|
# ── Step 8: Recent memories ─────────────────────────────────
|
|
header("Step 8: Recent Memories")
|
|
|
|
recent_a, elapsed = measure("get_recent_personal_memories", ms.get_recent_personal_memories,
|
|
"agent_alpha_1", limit=5)
|
|
ok(f"Recent personal memories for agent_alpha_1: {len(recent_a)} ({elapsed:.1f}ms)")
|
|
|
|
recent_team, elapsed = measure("get_recent_team_memories", ms.get_recent_team_memories,
|
|
"team_a", limit=5)
|
|
ok(f"Recent team memories for team_a: {len(recent_team)} ({elapsed:.1f}ms)")
|
|
|
|
# ── Performance Report ──────────────────────────────────────
|
|
header("Performance Report")
|
|
|
|
write_p50 = statistics.median(write_latencies)
|
|
write_p99 = sorted(write_latencies)[int(len(write_latencies) * 0.99)] if len(write_latencies) > 1 else write_latencies[0]
|
|
write_max = max(write_latencies)
|
|
|
|
search_p50 = statistics.median(search_latencies)
|
|
search_p99 = sorted(search_latencies)[int(len(search_latencies) * 0.99)] if len(search_latencies) > 1 else search_latencies[0]
|
|
search_max = max(search_latencies)
|
|
|
|
print(f" Write latency: P50={write_p50:.1f}ms P99={write_p99:.1f}ms Max={write_max:.1f}ms")
|
|
print(f" Search latency: P50={search_p50:.1f}ms P99={search_p99:.1f}ms Max={search_max:.1f}ms")
|
|
|
|
if write_p99 < 100:
|
|
ok("Write P99 < 100ms ✓")
|
|
else:
|
|
fail(f"Write P99 = {write_p99:.1f}ms (target < 100ms)")
|
|
|
|
if search_p99 < 50:
|
|
ok("Search P99 < 50ms ✓")
|
|
else:
|
|
warn(f"Search P99 = {search_p99:.1f}ms (target < 50ms, but embedding call dominates)")
|
|
|
|
# Memory usage
|
|
try:
|
|
import resource
|
|
rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # Linux: KB → MB
|
|
print(f" Process RSS: {rss_mb:.0f} MB")
|
|
if rss_mb < 1536:
|
|
ok("RSS < 1.5 GiB ✓")
|
|
else:
|
|
fail(f"RSS = {rss_mb:.0f} MB (target < 1536 MB)")
|
|
except:
|
|
warn("Could not measure RSS")
|
|
|
|
# Stats
|
|
header("System Stats")
|
|
stats, elapsed = measure("get_stats", ms.get_stats)
|
|
ok(f"Stats: {json.dumps(stats, indent=2)}")
|
|
|
|
# ── Summary ─────────────────────────────────────────────────
|
|
header("TEST SUMMARY")
|
|
print(f" Teams created: 2")
|
|
print(f" Agents created: 2")
|
|
print(f" Memories written: {len(write_latencies)}")
|
|
print(f" Searches executed: {len(search_latencies)}")
|
|
print(f" Team isolation: {'PASS' if (python_in_a and java_in_b and not java_in_a and not python_in_b) else 'CHECK MANUALLY'}")
|
|
print(f" Write P99: {write_p99:.1f}ms {'PASS' if write_p99 < 100 else 'WARN'}")
|
|
print(f" Search P99: {search_p99:.1f}ms {'PASS' if search_p99 < 50 else 'WARN'}")
|
|
print()
|
|
print(f"{GREEN}{BOLD}All validation tests completed!{RESET}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|