mirror of
https://github.com/khoaliber/khoj.git
synced 2026-03-06 13:22:12 +00:00
Improve agent creation safety checker with response schema, better prompt
This commit is contained in:
@@ -1252,6 +1252,7 @@ A: {{ "safe": "False", "reason": "The prompt contains sexual content that could
|
|||||||
Q: You are an astute financial analyst. Assess my financial situation and provide advice.
|
Q: You are an astute financial analyst. Assess my financial situation and provide advice.
|
||||||
A: {{ "safe": "True" }}
|
A: {{ "safe": "True" }}
|
||||||
|
|
||||||
|
# Actual:
|
||||||
Q: {prompt}
|
Q: {prompt}
|
||||||
A:
|
A:
|
||||||
""".strip()
|
""".strip()
|
||||||
@@ -1287,6 +1288,7 @@ A: {{ "safe": "False", "reason": "The prompt contains content that could be cons
|
|||||||
Q: You are a great analyst. Assess my financial situation and provide advice.
|
Q: You are a great analyst. Assess my financial situation and provide advice.
|
||||||
A: {{ "safe": "True" }}
|
A: {{ "safe": "True" }}
|
||||||
|
|
||||||
|
# Actual:
|
||||||
Q: {prompt}
|
Q: {prompt}
|
||||||
A:
|
A:
|
||||||
""".strip()
|
""".strip()
|
||||||
|
|||||||
@@ -321,13 +321,19 @@ async def acheck_if_safe_prompt(system_prompt: str, user: KhojUser = None, lax:
|
|||||||
is_safe = True
|
is_safe = True
|
||||||
reason = ""
|
reason = ""
|
||||||
|
|
||||||
|
class SafetyCheck(BaseModel):
|
||||||
|
safe: bool
|
||||||
|
reason: str
|
||||||
|
|
||||||
with timer("Chat actor: Check if safe prompt", logger):
|
with timer("Chat actor: Check if safe prompt", logger):
|
||||||
response = await send_message_to_model_wrapper(safe_prompt_check, user=user)
|
response = await send_message_to_model_wrapper(
|
||||||
|
safe_prompt_check, user=user, response_type="json_object", response_schema=SafetyCheck
|
||||||
|
)
|
||||||
|
|
||||||
response = response.strip()
|
response = response.strip()
|
||||||
try:
|
try:
|
||||||
response = json.loads(clean_json(response))
|
response = json.loads(clean_json(response))
|
||||||
is_safe = response.get("safe", "True") == "True"
|
is_safe = str(response.get("safe", "true")).lower() == "true"
|
||||||
if not is_safe:
|
if not is_safe:
|
||||||
reason = response.get("reason", "")
|
reason = response.get("reason", "")
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
Reference in New Issue
Block a user