初始化：记忆系统源代码上传（已脱敏）
- 排除 .env / *.bak / 内部运维文档（README_INTERNAL.html） - config.py 默认密码已替换为占位符 CHANGE_ME_* - init_db.sql 移除生产数据库用户 GRANT 段 - README.html 数据库用户名已脱敏 - 保留：源码 + 公网 API 文档 + 建表 SQL（无授权语句）
This commit is contained in:
zqf
@@ -0,0 +1 @@
|
||||
# Lifecycle management
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Scenario Aggregator - Group related memories into scenario blocks."""
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, List
|
||||
|
||||
from lifecycle.llm_parse import parse_llm_json_array
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SCENE_SYSTEM_PROMPT = """你是JSON生成器。直接输出JSON数组，禁止分析、思考、解释、讨论、markdown。
|
||||
|
||||
任务：将记忆按主题聚合成场景块。
|
||||
|
||||
规则：
|
||||
1. 将相关记忆归入同一场景
|
||||
2. 每个场景：name（名称）、summary（100字以内摘要）、memory_ids（ID列表）
|
||||
3. 最多10个场景
|
||||
4. 第一个字符必须是[，最后一个字符必须是]
|
||||
|
||||
输出格式：
|
||||
[{"name":"场景名","summary":"摘要","memory_ids":["id1","id2"]}]
|
||||
|
||||
只输出JSON，不要有任何其他文字。"""
|
||||
|
||||
|
||||
class ScenarioAggregator:
|
||||
"""Aggregate personal memories into scenario blocks."""
|
||||
|
||||
def __init__(self, mysql_store, llm_client):
|
||||
self.mysql = mysql_store
|
||||
self.llm = llm_client
|
||||
|
||||
def aggregate(self, agent_id: str, team_id: str,
|
||||
max_memories: int = 50) -> dict:
|
||||
"""Aggregate memories into scenarios using LLM."""
|
||||
if not self.llm.available:
|
||||
return {"error": "LLM not configured. Set LLM_API_URL, LLM_API_KEY, LLM_MODEL environment variables."}
|
||||
|
||||
memories = self.mysql.get_personal_memories_by_agent(agent_id, limit=max_memories)
|
||||
if not memories:
|
||||
return {"scenarios": [], "count": 0}
|
||||
|
||||
memory_lines = []
|
||||
for m in memories:
|
||||
mid = m["id"]
|
||||
content = m.get("content", "")[:100]
|
||||
memory_lines.append(f"[{mid}] {content}")
|
||||
user_prompt = "请将以下记忆聚合为JSON场景数组：\n\n" + "\n".join(memory_lines)
|
||||
|
||||
try:
|
||||
scenarios = parse_llm_json_array(
|
||||
self.llm, SCENE_SYSTEM_PROMPT, user_prompt,
|
||||
temperature=0.3, context="aggregate-scenes",
|
||||
max_tokens=2000,
|
||||
)
|
||||
except ValueError as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
stored = []
|
||||
for scene in scenarios:
|
||||
if not isinstance(scene, dict):
|
||||
logger.warning("Skipping non-dict item in scenarios: %r", scene)
|
||||
continue
|
||||
name = scene.get("name", "unnamed")
|
||||
summary = scene.get("summary", "")
|
||||
memory_ids = scene.get("memory_ids", [])
|
||||
valid_ids = [mid for mid in memory_ids if isinstance(mid, str) and len(mid) > 0]
|
||||
|
||||
scene_id = self.mysql.add_scenario(
|
||||
team_id=team_id,
|
||||
agent_id=agent_id,
|
||||
name=name,
|
||||
summary=summary,
|
||||
memory_ids=valid_ids,
|
||||
)
|
||||
stored.append({
|
||||
"id": scene_id,
|
||||
"name": name,
|
||||
"summary": summary,
|
||||
"memory_ids": valid_ids,
|
||||
})
|
||||
|
||||
return {"scenarios": stored, "count": len(stored)}
|
||||
|
||||
def get_scenarios(self, agent_id: str) -> List[dict]:
|
||||
return self.mysql.get_scenarios_by_agent(agent_id)
|
||||
|
||||
def delete_scenario(self, scenario_id: str, team_id: str) -> bool:
|
||||
scenario = self.mysql.get_scenario(scenario_id)
|
||||
if not scenario or scenario["team_id"] != team_id:
|
||||
return False
|
||||
return self.mysql.delete_scenario(scenario_id)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Memory Cleaner - Periodic cleanup of old/low-importance memories"""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryCleaner:
|
||||
"""Clean up old, low-importance memories."""
|
||||
|
||||
def __init__(self, mysql_store):
|
||||
self.mysql = mysql_store
|
||||
|
||||
def cleanup(self, team_id: str = None, max_age_days: int = 90,
|
||||
min_importance: float = 0.2) -> dict:
|
||||
"""
|
||||
Remove old memories with low importance scores.
|
||||
|
||||
Args:
|
||||
team_id: If set, only clean this team's memories
|
||||
max_age_days: Remove memories older than this
|
||||
min_importance: Remove memories with importance below this
|
||||
|
||||
Returns:
|
||||
Dict with counts of deleted memories
|
||||
"""
|
||||
personal_deleted = self.mysql.cleanup_personal_memories(
|
||||
team_id=team_id,
|
||||
max_age_days=max_age_days,
|
||||
min_importance=min_importance,
|
||||
)
|
||||
team_deleted = self.mysql.cleanup_team_memories(
|
||||
team_id=team_id,
|
||||
max_age_days=max_age_days,
|
||||
min_importance=min_importance,
|
||||
)
|
||||
|
||||
result = {
|
||||
"personal_deleted": personal_deleted,
|
||||
"team_deleted": team_deleted,
|
||||
"total_deleted": personal_deleted + team_deleted,
|
||||
}
|
||||
logger.info(f"Cleanup result: {result}")
|
||||
return result
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Working Memory Compressor"""
|
||||
import logging
|
||||
from typing import Optional, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryCompressor:
|
||||
"""Compress working memories when they exceed threshold."""
|
||||
|
||||
def __init__(self, mysql_store, redis_cache, embedding_service):
|
||||
self.mysql = mysql_store
|
||||
self.redis = redis_cache
|
||||
self.embedder = embedding_service
|
||||
|
||||
def compress(self, agent_id: str, max_items: int = 20,
|
||||
summary_callback: Optional[Callable] = None) -> str:
|
||||
"""
|
||||
Compress working memories into a single long-term memory.
|
||||
|
||||
If working memory exceeds max_items, take the oldest items,
|
||||
summarize them (via callback or simple concatenation), and
|
||||
store as a personal long-term memory.
|
||||
|
||||
Returns: Summary text of compressed memories.
|
||||
"""
|
||||
# Get agent info to find team_id
|
||||
agent = self.mysql.get_agent(agent_id)
|
||||
if not agent:
|
||||
raise ValueError(f"Agent {agent_id} not found")
|
||||
|
||||
items = self.redis.get_working_memories(agent_id, limit=100)
|
||||
if len(items) <= max_items:
|
||||
return {
|
||||
"compressed": 0,
|
||||
"remaining": len(items),
|
||||
"total_before": len(items),
|
||||
"target_count": max_items,
|
||||
"summary": None,
|
||||
}
|
||||
|
||||
# Take items beyond max_items (oldest)
|
||||
to_compress = items[max_items:]
|
||||
remaining = items[:max_items]
|
||||
|
||||
# Build summary
|
||||
contents = [item.get("content", "") for item in to_compress]
|
||||
if summary_callback:
|
||||
summary = summary_callback(contents)
|
||||
else:
|
||||
summary = " | ".join(contents[:50]) # Simple concat, limit 50 items
|
||||
|
||||
# Store as personal long-term memory
|
||||
embedding = self.embedder.embed(summary)
|
||||
from storage.vector_search import embedding_to_bytes
|
||||
emb_bytes = embedding_to_bytes(embedding)
|
||||
|
||||
self.mysql.add_personal_memory(
|
||||
agent_id=agent_id,
|
||||
team_id=agent["team_id"],
|
||||
content=summary,
|
||||
embedding=emb_bytes,
|
||||
importance=0.6, # Slightly elevated importance for compressed memories
|
||||
metadata={
|
||||
"source": "compression",
|
||||
"item_count": len(to_compress),
|
||||
"source_items": [
|
||||
{"content": item.get("content", "")[:200], "timestamp": item.get("timestamp")}
|
||||
for item in to_compress
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
# Replace working memory with remaining items
|
||||
self.redis.clear_working_memory(agent_id)
|
||||
for item in reversed(remaining): # lpush reverses order
|
||||
self.redis.add_working_memory(
|
||||
agent_id, item["content"],
|
||||
metadata=item.get("metadata", {}),
|
||||
)
|
||||
|
||||
logger.info(f"Compressed {len(to_compress)} working memories for agent {agent_id}")
|
||||
return {
|
||||
"compressed": len(to_compress),
|
||||
"remaining": len(remaining),
|
||||
"total_before": len(items),
|
||||
"target_count": max_items,
|
||||
"summary": summary,
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Fact Extractor - Extract structured atomic facts from working memory using LLM."""
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, List
|
||||
|
||||
from lifecycle.llm_parse import parse_llm_json_array
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FACT_SYSTEM_PROMPT = """你是JSON生成器。输入工作记忆，直接提取结构化事实。
|
||||
|
||||
严格规则：
|
||||
1. 禁止输出分析、推理、思考、解释
|
||||
2. 第一个字符必须是 [，最后一个字符必须是 ]
|
||||
3. 每条事实：content（50-200字）、importance（0.0-1.0）、category（分类）
|
||||
4. 最多20条，无值得提取的事实返回 []
|
||||
|
||||
输出格式（只输出JSON，不输出其他任何文字）：
|
||||
[{"content":"事实","importance":0.8,"category":"分类"}]"""
|
||||
|
||||
|
||||
class FactExtractor:
|
||||
"""Extract atomic facts from working memory using LLM."""
|
||||
|
||||
def __init__(self, mysql_store, redis_cache, llm_client, embedding_service):
|
||||
self.mysql = mysql_store
|
||||
self.redis = redis_cache
|
||||
self.llm = llm_client
|
||||
self.embedder = embedding_service
|
||||
|
||||
def extract(self, agent_id: str, team_id: str,
|
||||
max_memories: int = 20,
|
||||
delete_after: bool = False) -> dict:
|
||||
"""Extract structured facts from working memory."""
|
||||
if not self.llm.available:
|
||||
return {"error": "LLM not configured. Set LLM_API_URL, LLM_API_KEY, LLM_MODEL environment variables."}
|
||||
|
||||
items = self.redis.get_working_memories(agent_id, limit=max_memories)
|
||||
if not items:
|
||||
return {"facts": [], "count": 0}
|
||||
|
||||
memory_lines = []
|
||||
for i, item in enumerate(items):
|
||||
if not isinstance(item, dict):
|
||||
logger.warning('Skipping non-dict working memory item: %r', item)
|
||||
continue
|
||||
content = item.get("content", "")[:300]
|
||||
memory_lines.append(f"[{i+1}] {content}")
|
||||
user_prompt = "以下是需要提取事实的工作记忆：\n\n" + "\n".join(memory_lines)
|
||||
|
||||
try:
|
||||
facts = parse_llm_json_array(
|
||||
self.llm, FACT_SYSTEM_PROMPT, user_prompt,
|
||||
temperature=0.2, context="extract-facts",
|
||||
)
|
||||
except ValueError as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
stored = []
|
||||
for fact in facts:
|
||||
if not isinstance(fact, dict):
|
||||
logger.warning('Skipping non-dict fact from LLM: %r', fact)
|
||||
continue
|
||||
content = fact.get("content", "").strip()
|
||||
if not content or len(content) < 10:
|
||||
continue
|
||||
|
||||
importance = fact.get("importance", 0.6)
|
||||
if not isinstance(importance, (int, float)) or importance < 0 or importance > 1:
|
||||
importance = 0.6
|
||||
|
||||
category = fact.get("category", "")
|
||||
|
||||
embedding = self.embedder.embed(content)
|
||||
from storage.vector_search import embedding_to_bytes
|
||||
emb_bytes = embedding_to_bytes(embedding)
|
||||
|
||||
memory_id = self.mysql.add_personal_memory(
|
||||
agent_id=agent_id,
|
||||
team_id=team_id,
|
||||
content=content,
|
||||
embedding=emb_bytes,
|
||||
importance=float(importance),
|
||||
metadata={
|
||||
"source": "fact_extraction",
|
||||
"category": category,
|
||||
"extracted_from": "working_memory",
|
||||
},
|
||||
)
|
||||
stored.append({
|
||||
"id": memory_id,
|
||||
"content": content,
|
||||
"importance": importance,
|
||||
"category": category,
|
||||
})
|
||||
|
||||
if delete_after and stored:
|
||||
self.redis.clear_working_memory(agent_id)
|
||||
logger.info(f"Deleted working memories for {agent_id} after extracting {len(stored)} facts")
|
||||
|
||||
return {"facts": stored, "count": len(stored)}
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Shared LLM response parsing utilities for lifecycle modules."""
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_PARSE_RETRIES = 2 # 3 total attempts (1 initial + 2 retry)
|
||||
|
||||
RETRY_SYSTEM_PROMPT = (
|
||||
'只输出原始JSON。不要有任何其他文字、分析、思考、解释、markdown。\n'
|
||||
'输出格式示例：\n'
|
||||
'[{"name": "场景名", "summary": "摘要", "memory_ids": ["id1", "id2"]}]\n'
|
||||
'重要：你回复的第一个字符必须是 [，最后一个字符必须是 ]。'
|
||||
)
|
||||
|
||||
RETRY_OBJECT_PROMPT = (
|
||||
'只输出原始JSON对象。不要有任何其他文字、分析、思考、解释、markdown。\n'
|
||||
'输出格式：\n'
|
||||
'{"data": [{"name": "场景名", "summary": "摘要", "memory_ids": ["id1", "id2"]}]}\n'
|
||||
'重要：你回复的第一个字符必须是 {，最后一个字符必须是 }。'
|
||||
)
|
||||
|
||||
|
||||
def _strip_markdown_code_block(text: str) -> str:
|
||||
text = text.strip()
|
||||
if not text.startswith("```"):
|
||||
return text
|
||||
text = text.split("\n", 1)[1] if "\n" in text else text[3:]
|
||||
if text.endswith("```"):
|
||||
text = text[:-3].strip()
|
||||
if text.lower().startswith("json"):
|
||||
text = text[4:].strip()
|
||||
elif text.lower().startswith("json\n"):
|
||||
text = text[5:].strip()
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _strip_reasoning_prefix(text: str) -> str:
|
||||
"""Strip reasoning/analysis text that some models output before JSON.
|
||||
|
||||
Models like deepseek-v4-flash often output chain-of-thought analysis
|
||||
before the actual JSON. This function tries to find where the JSON
|
||||
actually starts by looking for lines that start with [ or { after
|
||||
stripping analysis text.
|
||||
"""
|
||||
text = text.strip()
|
||||
|
||||
# If text starts with [ or { it might already be JSON
|
||||
if text and text[0] in ('[', '{'):
|
||||
return text
|
||||
|
||||
# Try to find the last occurrence of a JSON-like pattern
|
||||
lines = text.split('\n')
|
||||
for i in range(len(lines) - 1, -1, -1):
|
||||
stripped = lines[i].strip()
|
||||
if stripped.startswith('[') or stripped.startswith('{'):
|
||||
candidate = '\n'.join(lines[i:])
|
||||
try:
|
||||
json.loads(candidate)
|
||||
return candidate
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
# Try finding first [ or { that starts a valid JSON
|
||||
for start_char in ['[', '{']:
|
||||
idx = text.find(start_char)
|
||||
while idx >= 0:
|
||||
candidate = text[idx:]
|
||||
end_char = ']' if start_char == '[' else '}'
|
||||
if candidate.rstrip().endswith(end_char):
|
||||
try:
|
||||
json.loads(candidate)
|
||||
return candidate
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
idx = text.find(start_char, idx + 1)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _find_json_string(text: str) -> Optional[str]:
|
||||
"""Find the first valid JSON array or object in text.
|
||||
|
||||
Iterates through ALL bracket-delimited segments, not just from the
|
||||
first bracket. This handles cases where the LLM outputs analysis
|
||||
text containing bracketed IDs before the actual JSON.
|
||||
"""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
# Fast path: entire text is valid JSON
|
||||
if (text.startswith("[") or text.startswith("{")) and text[-1] in ("}", "]"):
|
||||
try:
|
||||
json.loads(text)
|
||||
return text
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
|
||||
# Strip markdown code blocks
|
||||
cleaned = _strip_markdown_code_block(text)
|
||||
if cleaned != text:
|
||||
return _find_json_string(cleaned)
|
||||
|
||||
# Strip reasoning prefix
|
||||
cleaned = _strip_reasoning_prefix(text)
|
||||
if cleaned != text:
|
||||
result = _find_json_string(cleaned)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# Scan all bracket-delimited segments (for "[" / "]" and "{" / "}")
|
||||
for start_char, end_char in [("[", "]"), ("{", "}")]:
|
||||
search_start = 0
|
||||
while True:
|
||||
start_idx = text.find(start_char, search_start)
|
||||
if start_idx == -1:
|
||||
break
|
||||
depth = 0
|
||||
for i in range(start_idx, len(text)):
|
||||
if text[i] == start_char:
|
||||
depth += 1
|
||||
elif text[i] == end_char:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
candidate = text[start_idx:i + 1]
|
||||
try:
|
||||
json.loads(candidate)
|
||||
return candidate
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
# Move past this bracket pair and continue scanning
|
||||
search_start = i + 1
|
||||
break
|
||||
else:
|
||||
# No matching end bracket found
|
||||
search_start = start_idx + 1
|
||||
|
||||
# Regex fallback: find ALL bracket-delimited segments
|
||||
for pattern in [r"\[.*\]", r"\{.*\}"]:
|
||||
for match in re.finditer(pattern, text, re.DOTALL):
|
||||
candidate = match.group(0)
|
||||
try:
|
||||
json.loads(candidate)
|
||||
return candidate
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _try_fix_json(text: str) -> Optional[str]:
|
||||
"""Try to fix common JSON issues from LLM output."""
|
||||
# Remove trailing commas before } or ]
|
||||
text = re.sub(r",\s*}", "}", text)
|
||||
text = re.sub(r",\s*\]", "]", text)
|
||||
# Replace single quotes with double quotes only if no double quotes present
|
||||
if '"' not in text:
|
||||
text = text.replace("'", '"')
|
||||
# Remove BOM and zero-width characters
|
||||
text = text.replace("\ufeff", "").replace("\u200b", "")
|
||||
# Try parsing as-is first
|
||||
try:
|
||||
json.loads(text)
|
||||
return text
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Try fixing unescaped newlines inside JSON string values
|
||||
fixed = re.sub(r'(?<=: ")((?:[^"]|\\")*?)(\n)((?:[^"]|\\")*?(?="))', r'\1\\n\3', text)
|
||||
try:
|
||||
json.loads(fixed)
|
||||
return fixed
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Try wrapping in array if it looks like a bare object
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("{") and not stripped.startswith("["):
|
||||
wrapped = "[" + stripped + "]"
|
||||
try:
|
||||
json.loads(wrapped)
|
||||
return wrapped
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Try to extract JSON from truncated output: find last complete object in array
|
||||
if stripped.startswith("["):
|
||||
depth = 0
|
||||
objects = []
|
||||
obj_start = -1
|
||||
for i, ch in enumerate(stripped):
|
||||
if ch == "{":
|
||||
if depth == 0:
|
||||
obj_start = i
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0 and obj_start >= 0:
|
||||
objects.append(stripped[obj_start:i + 1])
|
||||
obj_start = -1
|
||||
if objects:
|
||||
candidate = "[" + ",".join(objects) + "]"
|
||||
try:
|
||||
json.loads(candidate)
|
||||
return candidate
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def parse_llm_json_array(llm_client, system_prompt, user_prompt,
|
||||
temperature=0.3, context="", max_tokens=4000):
|
||||
last_raw = ""
|
||||
total = 1 + MAX_PARSE_RETRIES
|
||||
t_start = time.time()
|
||||
for attempt in range(total):
|
||||
is_retry = attempt > 0
|
||||
|
||||
if is_retry:
|
||||
if attempt >= total - 1:
|
||||
# Final attempt: json_mode as last resort with wrapper object
|
||||
prompt = RETRY_OBJECT_PROMPT
|
||||
use_json_mode = True
|
||||
prefill_val = None
|
||||
else:
|
||||
# Intermediate retry: use prefill "["
|
||||
prompt = RETRY_SYSTEM_PROMPT
|
||||
use_json_mode = False
|
||||
prefill_val = "["
|
||||
else:
|
||||
# First attempt: prefill "[" to force JSON output immediately
|
||||
prompt = system_prompt
|
||||
use_json_mode = False
|
||||
prefill_val = "["
|
||||
|
||||
response = llm_client.chat(
|
||||
prompt, user_prompt,
|
||||
temperature=temperature, json_mode=use_json_mode,
|
||||
max_tokens=max_tokens,
|
||||
thinking={"type": "disabled"},
|
||||
prefill=prefill_val,
|
||||
)
|
||||
if not response:
|
||||
logger.warning("[%s] Attempt %d: LLM returned empty", context, attempt + 1)
|
||||
continue
|
||||
last_raw = response
|
||||
json_str = _find_json_string(response)
|
||||
if json_str:
|
||||
try:
|
||||
result = json.loads(json_str)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("[%s] Attempt %d: JSON parse error: %s", context, attempt + 1, e)
|
||||
continue
|
||||
if isinstance(result, list):
|
||||
elapsed = time.time() - t_start
|
||||
logger.info("[%s] Parsed JSON array in %.1fs (%d attempt(s), %d items)", context, elapsed, attempt + 1, len(result))
|
||||
return result
|
||||
if isinstance(result, dict):
|
||||
if "data" in result and isinstance(result["data"], list):
|
||||
elapsed = time.time() - t_start
|
||||
logger.info("[%s] Parsed JSON array in %.1fs (%d attempt(s), %d items)", context, elapsed, attempt + 1, len(result["data"]))
|
||||
return result["data"]
|
||||
logger.warning("[%s] Attempt %d: LLM returned dict, wrapping in list", context, attempt + 1)
|
||||
return [result]
|
||||
fixed = _try_fix_json(response)
|
||||
if fixed:
|
||||
try:
|
||||
result = json.loads(fixed)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(result, list):
|
||||
return result
|
||||
if isinstance(result, dict):
|
||||
if "data" in result and isinstance(result["data"], list):
|
||||
return result["data"]
|
||||
return [result]
|
||||
logger.warning(
|
||||
"[%s] Attempt %d/%d: No valid JSON array found (len=%d): %r",
|
||||
context, attempt + 1, total, len(response), response[:500]
|
||||
)
|
||||
logger.error("[%s] All %d attempts failed. Last raw: %r", context, total, last_raw[:500])
|
||||
raise ValueError("LLM failed to return valid JSON after retries")
|
||||
|
||||
|
||||
def parse_llm_json_object(llm_client, system_prompt, user_prompt,
|
||||
temperature=0.3, context="", max_tokens=4000):
|
||||
last_raw = ""
|
||||
total = 1 + MAX_PARSE_RETRIES
|
||||
t_start = time.time()
|
||||
for attempt in range(total):
|
||||
is_retry = attempt > 0
|
||||
|
||||
if is_retry:
|
||||
if attempt >= total - 1:
|
||||
# Final attempt: json_mode as last resort
|
||||
prompt = '只输出 JSON 对象。不要输出任何其他文字。\n输出格式：{"preferences":[],"habits":[],"expertise":[],"communication_style":"","summary":""}'
|
||||
use_json_mode = True
|
||||
prefill_val = None
|
||||
else:
|
||||
prompt = '只输出 JSON 对象。不要输出任何其他文字。\n禁止输出：分析、思考、解释、讨论、描述、markdown。'
|
||||
use_json_mode = False
|
||||
prefill_val = "{"
|
||||
else:
|
||||
# First attempt: prefill "{" to force JSON output
|
||||
prompt = system_prompt
|
||||
use_json_mode = False
|
||||
prefill_val = "{"
|
||||
|
||||
response = llm_client.chat(
|
||||
prompt, user_prompt,
|
||||
temperature=temperature, json_mode=use_json_mode,
|
||||
max_tokens=max_tokens,
|
||||
thinking={"type": "disabled"},
|
||||
prefill=prefill_val,
|
||||
)
|
||||
if not response:
|
||||
logger.warning("[%s] Attempt %d: LLM returned empty", context, attempt + 1)
|
||||
continue
|
||||
last_raw = response
|
||||
json_str = _find_json_string(response)
|
||||
if json_str:
|
||||
try:
|
||||
result = json.loads(json_str)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("[%s] Attempt %d: JSON parse error: %s", context, attempt + 1, e)
|
||||
continue
|
||||
if isinstance(result, dict):
|
||||
elapsed = time.time() - t_start
|
||||
logger.info("[%s] Parsed JSON object in %.1fs (%d attempt(s))", context, elapsed, attempt + 1)
|
||||
return result
|
||||
fixed = _try_fix_json(response)
|
||||
if fixed:
|
||||
try:
|
||||
result = json.loads(fixed)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(result, dict):
|
||||
return result
|
||||
logger.warning(
|
||||
"[%s] Attempt %d/%d: No valid JSON object found (len=%d): %r",
|
||||
context, attempt + 1, total, len(response), response[:500]
|
||||
)
|
||||
logger.error("[%s] All %d attempts failed. Last raw: %r", context, total, last_raw[:500])
|
||||
raise ValueError("LLM failed to return valid JSON after retries")
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Persona Generator - Generate user persona from scenarios and memories."""
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from lifecycle.llm_parse import parse_llm_json_object
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PERSONA_SYSTEM_PROMPT = """你是JSON生成器。直接输出JSON对象，禁止分析、思考、解释、markdown。
|
||||
|
||||
根据记忆和场景生成用户画像。
|
||||
|
||||
输出格式（第一个字符必须是{，最后一个字符必须是}）：
|
||||
{"preferences":["偏好1"],"habits":["习惯1"],"expertise":["擅长1"],"communication_style":"风格","summary":"100字以内摘要"}
|
||||
|
||||
只输出JSON，不要有任何其他文字。"""
|
||||
|
||||
|
||||
class PersonaGenerator:
|
||||
"""Generate user persona from memories and scenarios."""
|
||||
|
||||
def __init__(self, mysql_store, llm_client):
|
||||
self.mysql = mysql_store
|
||||
self.llm = llm_client
|
||||
|
||||
def generate(self, agent_id: str, team_id: str,
|
||||
max_items: int = 30) -> dict:
|
||||
if not self.llm.available:
|
||||
return {"error": "LLM not configured. Set LLM_API_URL, LLM_API_KEY, LLM_MODEL environment variables."}
|
||||
|
||||
memories = self.mysql.get_personal_memories_by_agent(agent_id, limit=max_items)
|
||||
scenarios = self.mysql.get_scenarios_by_agent(agent_id)
|
||||
|
||||
if not memories and not scenarios:
|
||||
return {"error": "No memories or scenarios found for this agent"}
|
||||
|
||||
context_parts = []
|
||||
if memories:
|
||||
mem_lines = [f"- {m.get('content', '')[:100]}" for m in memories]
|
||||
context_parts.append("历史记忆：\n" + "\n".join(mem_lines))
|
||||
if scenarios:
|
||||
scene_lines = [f"- [{s.get('name', '')}] {s.get('summary', '')}" for s in scenarios]
|
||||
context_parts.append("已识别场景：\n" + "\n".join(scene_lines))
|
||||
|
||||
user_prompt = "请根据以下内容生成用户画像JSON：\n\n" + "\n\n".join(context_parts)
|
||||
|
||||
try:
|
||||
persona = parse_llm_json_object(
|
||||
self.llm, PERSONA_SYSTEM_PROMPT, user_prompt,
|
||||
temperature=0.3, context="generate-persona",
|
||||
max_tokens=2000,
|
||||
)
|
||||
except ValueError as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
if not isinstance(persona, dict):
|
||||
logger.warning("LLM returned non-dict persona: %r", persona)
|
||||
return {"error": "LLM returned invalid persona format"}
|
||||
|
||||
pid = self.mysql.add_persona(
|
||||
team_id=team_id,
|
||||
agent_id=agent_id,
|
||||
preferences=persona.get("preferences", []),
|
||||
habits=persona.get("habits", []),
|
||||
expertise=persona.get("expertise", []),
|
||||
communication_style=persona.get("communication_style", ""),
|
||||
summary=persona.get("summary", ""),
|
||||
)
|
||||
persona["id"] = pid
|
||||
return {"persona": persona}
|
||||
|
||||
def get_persona(self, agent_id: str):
|
||||
return self.mysql.get_persona_by_agent(agent_id)
|
||||
|
||||
def delete_persona(self, persona_id: str, team_id: str) -> bool:
|
||||
persona = self.mysql.get_persona(persona_id)
|
||||
if not persona or persona["team_id"] != team_id:
|
||||
return False
|
||||
return self.mysql.delete_persona(persona_id)
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Pipeline Automation - Auto-trigger compression and cleanup based on configurable rules."""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PipelineConfig:
|
||||
"""Pipeline configuration for a team."""
|
||||
|
||||
def __init__(self, team_id: str, compress_every_n: int = 0,
|
||||
compress_target_count: int = 5,
|
||||
cleanup_idle_days: int = 0,
|
||||
cleanup_min_importance: float = 0.2,
|
||||
enabled: bool = True, **kwargs):
|
||||
self.team_id = team_id
|
||||
self.compress_every_n = compress_every_n # 0 = disabled
|
||||
self.compress_target_count = compress_target_count
|
||||
self.cleanup_idle_days = cleanup_idle_days # 0 = disabled
|
||||
self.cleanup_min_importance = cleanup_min_importance
|
||||
self.enabled = enabled
|
||||
self.warmup_max_memories = kwargs.get("warmup_max_memories", 0) # 0 = disabled
|
||||
self.warmup_compress_every_n = kwargs.get("warmup_compress_every_n", 1)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"team_id": self.team_id,
|
||||
"compress_every_n": self.compress_every_n,
|
||||
"compress_target_count": self.compress_target_count,
|
||||
"cleanup_idle_days": self.cleanup_idle_days,
|
||||
"cleanup_min_importance": self.cleanup_min_importance,
|
||||
"enabled": self.enabled,
|
||||
"warmup_max_memories": self.warmup_max_memories,
|
||||
"warmup_compress_every_n": self.warmup_compress_every_n,
|
||||
}
|
||||
|
||||
|
||||
class Pipeline:
|
||||
"""Auto-pipeline: trigger compression and cleanup based on rules."""
|
||||
|
||||
def __init__(self, mysql_store, redis_cache, compressor, cleaner):
|
||||
self.mysql = mysql_store
|
||||
self.redis = redis_cache
|
||||
self.compressor = compressor
|
||||
self.cleaner = cleaner
|
||||
|
||||
def get_config(self, team_id: str) -> Optional[PipelineConfig]:
|
||||
row = self.mysql.get_pipeline_config(team_id)
|
||||
if not row:
|
||||
return None
|
||||
return PipelineConfig(
|
||||
team_id=row["team_id"],
|
||||
compress_every_n=row.get("compress_every_n", 0),
|
||||
compress_target_count=row.get("compress_target_count", 5),
|
||||
cleanup_idle_days=row.get("cleanup_idle_days", 0),
|
||||
cleanup_min_importance=row.get("cleanup_min_importance", 0.2),
|
||||
enabled=bool(row.get("enabled", 1)),
|
||||
warmup_max_memories=row.get("warmup_max_memories", 0),
|
||||
warmup_compress_every_n=row.get("warmup_compress_every_n", 1),
|
||||
)
|
||||
|
||||
def set_config(self, team_id: str, **kwargs) -> PipelineConfig:
|
||||
existing = self.mysql.get_pipeline_config(team_id)
|
||||
if existing:
|
||||
self.mysql.update_pipeline_config(team_id, **kwargs)
|
||||
else:
|
||||
self.mysql.create_pipeline_config(team_id, **kwargs)
|
||||
return self.get_config(team_id)
|
||||
|
||||
def check_after_wm_write(self, agent_id: str) -> Optional[dict]:
|
||||
agent = self.mysql.get_agent(agent_id)
|
||||
if not agent:
|
||||
return None
|
||||
team_id = agent["team_id"]
|
||||
config = self.get_config(team_id)
|
||||
if not config or not config.enabled:
|
||||
return None
|
||||
if config.compress_every_n <= 0:
|
||||
return None
|
||||
wm_count = self.redis.get_working_memory_count(agent_id)
|
||||
# Warm-up: use lower threshold for new agents
|
||||
effective_threshold = config.compress_every_n
|
||||
if config.warmup_max_memories > 0:
|
||||
personal_count = len(self.mysql.get_personal_memories_by_agent(agent_id, limit=config.warmup_max_memories + 1))
|
||||
if personal_count < config.warmup_max_memories:
|
||||
effective_threshold = config.warmup_compress_every_n
|
||||
if wm_count < effective_threshold:
|
||||
return None
|
||||
logger.info(f"Auto-compress triggered for {agent_id}: WM {wm_count} >= {config.compress_every_n}")
|
||||
try:
|
||||
result = self.compressor.compress(agent_id, max_items=config.compress_target_count)
|
||||
result["auto_triggered"] = True
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Auto-compress failed for {agent_id}: {e}")
|
||||
return None
|
||||
|
||||
def run_cleanup(self, team_id: str = None) -> Optional[dict]:
|
||||
configs = []
|
||||
if team_id:
|
||||
config = self.get_config(team_id)
|
||||
if config and config.enabled and config.cleanup_idle_days > 0:
|
||||
configs.append(config)
|
||||
else:
|
||||
all_configs = self.mysql.get_all_pipeline_configs()
|
||||
for row in all_configs:
|
||||
config = PipelineConfig(
|
||||
team_id=row["team_id"],
|
||||
compress_every_n=row.get("compress_every_n", 0),
|
||||
compress_target_count=row.get("compress_target_count", 5),
|
||||
cleanup_idle_days=row.get("cleanup_idle_days", 0),
|
||||
cleanup_min_importance=row.get("cleanup_min_importance", 0.2),
|
||||
enabled=bool(row.get("enabled", 1)),
|
||||
warmup_max_memories=row.get("warmup_max_memories", 0),
|
||||
warmup_compress_every_n=row.get("warmup_compress_every_n", 1),
|
||||
)
|
||||
if config.enabled and config.cleanup_idle_days > 0:
|
||||
configs.append(config)
|
||||
if not configs:
|
||||
return None
|
||||
results = []
|
||||
for config in configs:
|
||||
try:
|
||||
result = self.cleaner.cleanup(
|
||||
team_id=config.team_id,
|
||||
max_age_days=config.cleanup_idle_days,
|
||||
min_importance=config.cleanup_min_importance,
|
||||
)
|
||||
result["team_id"] = config.team_id
|
||||
result["auto_triggered"] = True
|
||||
results.append(result)
|
||||
logger.info(f"Auto-cleanup for {config.team_id}: {result}")
|
||||
except Exception as e:
|
||||
logger.error(f"Auto-cleanup failed for {config.team_id}: {e}")
|
||||
results.append({"team_id": config.team_id, "error": str(e)})
|
||||
return {"results": results}
|
||||
Reference in New Issue
Block a user