diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx index 5d87f7ea..ef074ae2 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.tsx +++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx @@ -31,6 +31,7 @@ import { Shapes, Trash, Toolbox, + Browser, } from "@phosphor-icons/react"; import DOMPurify from "dompurify"; @@ -333,6 +334,10 @@ function chooseIconFromHeader(header: string, iconColor: string) { return ; } + if (compareHeader.includes("operating")) { + return ; + } + return ; } @@ -342,10 +347,27 @@ export function TrainOfThought(props: TrainOfThoughtProps) { let header = extractedHeader ? extractedHeader[1] : ""; const iconColor = props.primary ? convertColorToTextClass(props.agentColor) : "text-gray-500"; const icon = chooseIconFromHeader(header, iconColor); - let markdownRendered = DOMPurify.sanitize(md.render(props.message)); + let message = props.message; - // Remove any header tags from markdownRendered + // Render screenshot image in screenshot action message + let screenshotData = null; + try { + const jsonMatch = message.match(/\{"action": "screenshot".*\}/); + if (jsonMatch) { + screenshotData = JSON.parse(jsonMatch[0]); + const screenshotHtmlString = `State of browser`; + message = message.replace(jsonMatch[0], `Screenshot\n\n${screenshotHtmlString}`); + } + } catch (e) { + console.error("Failed to parse screenshot data", e); + } + + // Render the sanitized train of thought as markdown + let markdownRendered = DOMPurify.sanitize(md.render(message)); + + // Remove any header tags from the rendered markdown markdownRendered = markdownRendered.replace(//g, ""); + return (
str: return "\n- ".join(compiled_response) +async def render_claude_response(response_content: list[BetaContentBlock], page: Page) -> str: + """ + Share the response from Anthropic AI model to be rendered by the client. + """ + compiled_response = [""] + for block in deepcopy(response_content): + if block.type == "text": + compiled_response.append(block.text) + elif block.type == "tool_use": + if hasattr(block, "name") and block.name == "goto": + block_input = {"action": block.name, "url": block.input.get("url")} + elif hasattr(block, "name") and block.name == "back": + block_input = {"action": block.name} + else: + block_input = block.input + + if block_input.get("action") == "screenshot": + screenshot_base64 = await get_screenshot(page) + block_input["image"] = f"data:image/webp;base64,{screenshot_base64}" + + compiled_response.append(f"**Action**: {json.dumps(block_input)}") + elif block.type == "thinking": + compiled_response.append(f"**Thought**: {block.thinking}") + return "\n- ".join(compiled_response) + + async def get_screenshot(page: Page): """ Take a viewport screenshot using Playwright and return as base64 encoded webp image.