From 258b5a0372f00040e1753e4651f84c9a047acb4a Mon Sep 17 00:00:00 2001
From: Debanjum <debanjum@gmail.com>
Date: Fri, 9 May 2025 14:49:37 -0600
Subject: [PATCH] Show operator screenshots with reasoning in train of thought
 on web app

---
 .../app/components/chatMessage/chatMessage.tsx    | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx
index 794105c1..2e4c600d 100644
--- a/src/interface/web/app/components/chatMessage/chatMessage.tsx
+++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx
@@ -350,13 +350,18 @@ export function TrainOfThought(props: TrainOfThoughtProps) {
     let message = props.message;
 
     // Render screenshot image in screenshot action message
-    let screenshotData = null;
+    let jsonMessage = null;
     try {
-        const jsonMatch = message.match(/\{.*("action": "screenshot"|"type": "screenshot").*\}/);
+        const jsonMatch = message.match(
+            /\{.*("action": "screenshot"|"type": "screenshot"|"image": "data:image\/.*").*\}/,
+        );
         if (jsonMatch) {
-            screenshotData = JSON.parse(jsonMatch[0]);
-            const screenshotHtmlString = `<img src="${screenshotData.image}" alt="State of browser" class="max-w-full" />`;
-            message = message.replace(jsonMatch[0], `Screenshot\n\n${screenshotHtmlString}`);
+            jsonMessage = JSON.parse(jsonMatch[0]);
+            const screenshotHtmlString = `<img src="${jsonMessage.image}" alt="State of environment" class="max-w-full" />`;
+            message = message.replace(
+                `:\n**Action**: ${jsonMatch[0]}`,
+                `\n\n- ${jsonMessage.text}\n${screenshotHtmlString}`,
+            );
         }
     } catch (e) {
         console.error("Failed to parse screenshot data", e);