初始化：记忆系统源代码上传（已脱敏）
- 排除 .env / *.bak / 内部运维文档（README_INTERNAL.html） - config.py 默认密码已替换为占位符 CHANGE_ME_* - init_db.sql 移除生产数据库用户 GRANT 段 - README.html 数据库用户名已脱敏 - 保留：源码 + 公网 API 文档 + 建表 SQL（无授权语句）
This commit is contained in:
zqf
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Memory System Validation Tests
|
||||
|
||||
Tests:
|
||||
1. Create two teams (team_a, team_b)
|
||||
2. Create agents in each team
|
||||
3. Write personal memories with team-specific content
|
||||
4. Semantic search with team isolation verification
|
||||
5. Performance benchmarks (write latency, search latency, memory usage)
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import statistics
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from config import Config
|
||||
from memory_system import MemorySystem
|
||||
|
||||
# ANSI colors
|
||||
GREEN = "\033[92m"
|
||||
RED = "\033[91m"
|
||||
YELLOW = "\033[93m"
|
||||
CYAN = "\033[96m"
|
||||
RESET = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
|
||||
def header(text):
|
||||
print(f"\n{BOLD}{CYAN}{'='*60}{RESET}")
|
||||
print(f"{BOLD}{CYAN}{text}{RESET}")
|
||||
print(f"{BOLD}{CYAN}{'='*60}{RESET}")
|
||||
|
||||
def ok(msg):
|
||||
print(f" {GREEN}✓{RESET} {msg}")
|
||||
|
||||
def fail(msg):
|
||||
print(f" {RED}✗{RESET} {msg}")
|
||||
|
||||
def warn(msg):
|
||||
print(f" {YELLOW}⚠{RESET} {msg}")
|
||||
|
||||
def measure(label, func, *args, **kwargs):
|
||||
"""Run func and return (result, elapsed_ms)."""
|
||||
start = time.perf_counter()
|
||||
result = func(*args, **kwargs)
|
||||
elapsed = (time.perf_counter() - start) * 1000
|
||||
return result, elapsed
|
||||
|
||||
|
||||
def main():
|
||||
cfg = Config()
|
||||
ms = MemorySystem(cfg)
|
||||
|
||||
# ── Health checks ────────────────────────────────────────────
|
||||
header("Health Checks")
|
||||
|
||||
if ms.embedder.health_check():
|
||||
ok("BGE embedding service is alive")
|
||||
else:
|
||||
fail("BGE embedding service not responding")
|
||||
sys.exit(1)
|
||||
|
||||
if ms.redis.health_check():
|
||||
ok("Redis connection OK")
|
||||
else:
|
||||
fail("Redis connection failed")
|
||||
sys.exit(1)
|
||||
|
||||
ok("MySQL connection OK (implicit via queries)")
|
||||
|
||||
# ── Step 1: Create teams ─────────────────────────────────────
|
||||
header("Step 1: Create Teams")
|
||||
|
||||
# Clean up first (idempotent)
|
||||
try:
|
||||
ms.delete_team("team_a")
|
||||
ms.delete_team("team_b")
|
||||
except:
|
||||
pass
|
||||
|
||||
team_a, elapsed = measure("create_team", ms.create_team, "team_a", "Team Alpha", "First test team")
|
||||
ok(f"Created team_a: {team_a['name']} ({elapsed:.1f}ms)")
|
||||
|
||||
team_b, elapsed = measure("create_team", ms.create_team, "team_b", "Team Beta", "Second test team")
|
||||
ok(f"Created team_b: {team_b['name']} ({elapsed:.1f}ms)")
|
||||
|
||||
# ── Step 2: Create agents ────────────────────────────────────
|
||||
header("Step 2: Create Agents")
|
||||
|
||||
agent_a, elapsed = measure("create_agent", ms.create_agent, "agent_alpha_1", "team_a", "Alpha Agent", "developer")
|
||||
ok(f"Created agent_alpha_1 in team_a ({elapsed:.1f}ms)")
|
||||
|
||||
agent_b, elapsed = measure("create_agent", ms.create_agent, "agent_beta_1", "team_b", "Beta Agent", "developer")
|
||||
ok(f"Created agent_beta_1 in team_b ({elapsed:.1f}ms)")
|
||||
|
||||
# ── Step 3: Write personal memories ──────────────────────────
|
||||
header("Step 3: Write Personal Memories")
|
||||
|
||||
write_latencies = []
|
||||
|
||||
mem_a, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
||||
"agent_alpha_1", "用户的编程偏好是 Python，喜欢用 Flask 和 FastAPI 框架",
|
||||
importance=0.8, metadata={"topic": "preference"})
|
||||
write_latencies.append(elapsed)
|
||||
ok(f"team_a memory: '{mem_a['content'][:40]}...' ({elapsed:.1f}ms)")
|
||||
|
||||
mem_b, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
||||
"agent_beta_1", "用户的编程偏好是 Java，喜欢用 Spring Boot 框架",
|
||||
importance=0.8, metadata={"topic": "preference"})
|
||||
write_latencies.append(elapsed)
|
||||
ok(f"team_b memory: '{mem_b['content'][:40]}...' ({elapsed:.1f}ms)")
|
||||
|
||||
# Add more memories for richer testing
|
||||
for i, content in enumerate([
|
||||
"团队每周一开例会",
|
||||
"项目截止日期是下个月底",
|
||||
"用户不喜欢加班",
|
||||
]):
|
||||
_, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
||||
"agent_alpha_1", content, importance=0.5 + i * 0.1)
|
||||
write_latencies.append(elapsed)
|
||||
|
||||
for i, content in enumerate([
|
||||
"团队使用 Jenkins 做 CI/CD",
|
||||
"数据库用的是 PostgreSQL",
|
||||
"代码审查需要至少两人通过",
|
||||
]):
|
||||
_, elapsed = measure("add_personal_memory", ms.add_personal_memory,
|
||||
"agent_beta_1", content, importance=0.5 + i * 0.1)
|
||||
write_latencies.append(elapsed)
|
||||
|
||||
ok(f"Wrote {len(write_latencies)} memories total")
|
||||
|
||||
# ── Step 4: Write team memories ──────────────────────────────
|
||||
header("Step 4: Write Team Shared Memories")
|
||||
|
||||
tm_a, elapsed = measure("add_team_memory", ms.add_team_memory,
|
||||
"team_a", "团队技术栈: Python, Flask, MySQL, Redis",
|
||||
importance=0.9, category="tech_stack")
|
||||
write_latencies.append(elapsed)
|
||||
ok(f"team_a shared: '{tm_a['content'][:40]}...' ({elapsed:.1f}ms)")
|
||||
|
||||
tm_b, elapsed = measure("add_team_memory", ms.add_team_memory,
|
||||
"team_b", "团队技术栈: Java, Spring Boot, PostgreSQL, Kafka",
|
||||
importance=0.9, category="tech_stack")
|
||||
write_latencies.append(elapsed)
|
||||
ok(f"team_b shared: '{tm_b['content'][:40]}...' ({elapsed:.1f}ms)")
|
||||
|
||||
# ── Step 5: Semantic search with isolation ───────────────────
|
||||
header("Step 5: Semantic Search + Team Isolation")
|
||||
|
||||
search_latencies = []
|
||||
|
||||
# Search team_a for "编程偏好"
|
||||
results_a, elapsed = measure("search_personal_memories", ms.search_personal_memories,
|
||||
"agent_alpha_1", "编程偏好", limit=5)
|
||||
search_latencies.append(elapsed)
|
||||
ok(f"team_a search '编程偏好' → {len(results_a)} results ({elapsed:.1f}ms)")
|
||||
for r in results_a:
|
||||
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
||||
|
||||
# Search team_b for "编程偏好"
|
||||
results_b, elapsed = measure("search_personal_memories", ms.search_personal_memories,
|
||||
"agent_beta_1", "编程偏好", limit=5)
|
||||
search_latencies.append(elapsed)
|
||||
ok(f"team_b search '编程偏好' → {len(results_b)} results ({elapsed:.1f}ms)")
|
||||
for r in results_b:
|
||||
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
||||
|
||||
# Verify isolation
|
||||
header("Step 6: Cross-Team Isolation Verification")
|
||||
|
||||
a_contents = [r["content"] for r in results_a]
|
||||
b_contents = [r["content"] for r in results_b]
|
||||
|
||||
python_in_a = any("Python" in c for c in a_contents)
|
||||
java_in_a = any("Java" in c for c in a_contents)
|
||||
python_in_b = any("Python" in c for c in b_contents)
|
||||
java_in_b = any("Java" in c for c in b_contents)
|
||||
|
||||
if python_in_a and not java_in_a:
|
||||
ok("team_a returns Python (not Java) ✓")
|
||||
elif not a_contents:
|
||||
warn("team_a search returned no results")
|
||||
else:
|
||||
fail(f"team_a isolation issue: found Java={java_in_a}, Python={python_in_a}")
|
||||
|
||||
if java_in_b and not python_in_b:
|
||||
ok("team_b returns Java (not Python) ✓")
|
||||
elif not b_contents:
|
||||
warn("team_b search returned no results")
|
||||
else:
|
||||
fail(f"team_b isolation issue: found Python={python_in_b}, Java={java_in_b}")
|
||||
|
||||
# Also verify team shared memories
|
||||
team_search_a, elapsed = measure("search_team_memories", ms.search_team_memories,
|
||||
"team_a", "技术栈", limit=3)
|
||||
search_latencies.append(elapsed)
|
||||
ok(f"team_a shared search '技术栈' → {len(team_search_a)} results ({elapsed:.1f}ms)")
|
||||
for r in team_search_a:
|
||||
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
||||
|
||||
team_search_b, elapsed = measure("search_team_memories", ms.search_team_memories,
|
||||
"team_b", "技术栈", limit=3)
|
||||
search_latencies.append(elapsed)
|
||||
ok(f"team_b shared search '技术栈' → {len(team_search_b)} results ({elapsed:.1f}ms)")
|
||||
for r in team_search_b:
|
||||
print(f" score={r['score']:.4f} | {r['content'][:60]}")
|
||||
|
||||
# ── Step 7: Working memory ──────────────────────────────────
|
||||
header("Step 7: Working Memory (Redis)")
|
||||
|
||||
wm, elapsed = measure("add_working_memory", ms.add_working_memory,
|
||||
"agent_alpha_1", "刚刚和用户讨论了部署方案")
|
||||
ok(f"Added working memory ({elapsed:.1f}ms)")
|
||||
|
||||
wm_items, elapsed = measure("get_working_memories", ms.get_working_memories,
|
||||
"agent_alpha_1", limit=10)
|
||||
ok(f"Retrieved {len(wm_items)} working memories ({elapsed:.1f}ms)")
|
||||
|
||||
# ── Step 8: Recent memories ─────────────────────────────────
|
||||
header("Step 8: Recent Memories")
|
||||
|
||||
recent_a, elapsed = measure("get_recent_personal_memories", ms.get_recent_personal_memories,
|
||||
"agent_alpha_1", limit=5)
|
||||
ok(f"Recent personal memories for agent_alpha_1: {len(recent_a)} ({elapsed:.1f}ms)")
|
||||
|
||||
recent_team, elapsed = measure("get_recent_team_memories", ms.get_recent_team_memories,
|
||||
"team_a", limit=5)
|
||||
ok(f"Recent team memories for team_a: {len(recent_team)} ({elapsed:.1f}ms)")
|
||||
|
||||
# ── Performance Report ──────────────────────────────────────
|
||||
header("Performance Report")
|
||||
|
||||
write_p50 = statistics.median(write_latencies)
|
||||
write_p99 = sorted(write_latencies)[int(len(write_latencies) * 0.99)] if len(write_latencies) > 1 else write_latencies[0]
|
||||
write_max = max(write_latencies)
|
||||
|
||||
search_p50 = statistics.median(search_latencies)
|
||||
search_p99 = sorted(search_latencies)[int(len(search_latencies) * 0.99)] if len(search_latencies) > 1 else search_latencies[0]
|
||||
search_max = max(search_latencies)
|
||||
|
||||
print(f" Write latency: P50={write_p50:.1f}ms P99={write_p99:.1f}ms Max={write_max:.1f}ms")
|
||||
print(f" Search latency: P50={search_p50:.1f}ms P99={search_p99:.1f}ms Max={search_max:.1f}ms")
|
||||
|
||||
if write_p99 < 100:
|
||||
ok("Write P99 < 100ms ✓")
|
||||
else:
|
||||
fail(f"Write P99 = {write_p99:.1f}ms (target < 100ms)")
|
||||
|
||||
if search_p99 < 50:
|
||||
ok("Search P99 < 50ms ✓")
|
||||
else:
|
||||
warn(f"Search P99 = {search_p99:.1f}ms (target < 50ms, but embedding call dominates)")
|
||||
|
||||
# Memory usage
|
||||
try:
|
||||
import resource
|
||||
rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # Linux: KB → MB
|
||||
print(f" Process RSS: {rss_mb:.0f} MB")
|
||||
if rss_mb < 1536:
|
||||
ok("RSS < 1.5 GiB ✓")
|
||||
else:
|
||||
fail(f"RSS = {rss_mb:.0f} MB (target < 1536 MB)")
|
||||
except:
|
||||
warn("Could not measure RSS")
|
||||
|
||||
# Stats
|
||||
header("System Stats")
|
||||
stats, elapsed = measure("get_stats", ms.get_stats)
|
||||
ok(f"Stats: {json.dumps(stats, indent=2)}")
|
||||
|
||||
# ── Summary ─────────────────────────────────────────────────
|
||||
header("TEST SUMMARY")
|
||||
print(f" Teams created: 2")
|
||||
print(f" Agents created: 2")
|
||||
print(f" Memories written: {len(write_latencies)}")
|
||||
print(f" Searches executed: {len(search_latencies)}")
|
||||
print(f" Team isolation: {'PASS' if (python_in_a and java_in_b and not java_in_a and not python_in_b) else 'CHECK MANUALLY'}")
|
||||
print(f" Write P99: {write_p99:.1f}ms {'PASS' if write_p99 < 100 else 'WARN'}")
|
||||
print(f" Search P99: {search_p99:.1f}ms {'PASS' if search_p99 < 50 else 'WARN'}")
|
||||
print()
|
||||
print(f"{GREEN}{BOLD}All validation tests completed!{RESET}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user