feat(web-integration): support logScreenshot api for agent (#813)

* feat(web-integration): support logScreenshot api for agent

* docs(core): update api description

* docs(site): fix error

* feat(web-integration): title is also optional

* docs(site): fix

---------

Co-authored-by: yutao <[email protected]>

Author: quanru

apps/report/src/components/detail-panel.tsx

@@ -1,7 +1,6 @@
 'use client';
 import './detail-panel.less';
 import { useExecutionDump } from '@/components/store';
-import { filterBase64Value, timeStr } from '@/utils';
 import {
   CameraOutlined,
   FileTextOutlined,
@@ -12,6 +11,7 @@ import type {
   ExecutionTaskInsightLocate,
   ExecutionTaskPlanning,
 } from '@midscene/core';
+import { filterBase64Value, timeStr } from '@midscene/visualizer';
 import { Blackboard, Player } from '@midscene/visualizer';
 import { ConfigProvider, Segmented } from 'antd';
 import { useEffect, useState } from 'react';

apps/report/src/components/detail-side.less

@@ -10,13 +10,19 @@
     margin-bottom: 4px;
   }
 
-  .ant-tag{
+  .ant-tag {
     margin-top: 2px;
   }
 
+
+  .log-content {
+    padding: 20px;
+  }
+
   .meta-kv {
     padding: @side-vertical-spacing @side-horizontal-padding calc(@side-vertical-spacing + 4px);
     overflow: hidden;
+
     .meta {
       box-sizing: border-box;
       padding: 2px 0;
@@ -26,13 +32,13 @@
       line-height: 1.5;
 
       .meta-key {
-        width:100px;
+        width: 100px;
         text-align: right;
         padding-right: 16px;
         flex-shrink: 0;
         // color: #999;
       }
-      
+
       .meta-value {
         flex: 1;
       }
@@ -42,7 +48,7 @@
   .item-list {
     padding: @side-vertical-spacing @side-horizontal-padding;
 
-    cursor:default;
+    cursor: default;
     margin-bottom: 10px;
 
     .item {
@@ -61,11 +67,12 @@
 
     .item-highlight {
       color: #FFF;
+
       .subtitle {
         color: #CCC;
       }
     }
-    
+
     .item-extra {
       position: absolute;
       right: @side-horizontal-padding;
@@ -76,7 +83,7 @@
     .title-right-padding {
       padding-right: 15px;
     }
-  
+
     .title {
       font-size: 18px;
       font-weight: bold;
@@ -100,7 +107,7 @@
     .description {
       margin-top: @subSpacing;
     }
-  
+
     .description-content {
       font-size: 14px;
       margin-top: @side-horizontal-padding;
@@ -128,4 +135,4 @@
   .item-list-space-up {
     margin-top: @side-vertical-spacing;
   }
-}
+}

apps/report/src/components/detail-side.tsx

@@ -1,7 +1,7 @@
 /* eslint-disable max-lines */
 'use client';
 import './detail-side.less';
-import { timeStr } from '@/utils';
+import { timeStr } from '@midscene/visualizer';
 import { paramStr, typeStr } from '@midscene/web/ui-utils';
 
 import { RadiusSettingOutlined } from '@ant-design/icons';
@@ -221,11 +221,11 @@ const DetailSide = (): JSX.Element => {
     ],
   });
 
-  let taskParam: JSX.Element | null = null;
+  let taskInput: JSX.Element | null = null;
   if (task?.type === 'Planning') {
     const planningTask = task as ExecutionTaskPlanning;
     if (planningTask.param?.userInstruction) {
-      taskParam = MetaKV({
+      taskInput = MetaKV({
         data: [
           { key: 'type', content: (task && typeStr(task)) || '' },
           {
@@ -239,7 +239,7 @@ const DetailSide = (): JSX.Element => {
         ],
       });
     } else {
-      taskParam = MetaKV({
+      taskInput = MetaKV({
         data: [
           { key: 'type', content: (task && typeStr(task)) || '' },
           {
@@ -250,7 +250,7 @@ const DetailSide = (): JSX.Element => {
       });
     }
   } else if (task?.type === 'Insight') {
-    taskParam = MetaKV({
+    taskInput = MetaKV({
       data: [
         { key: 'type', content: (task && typeStr(task)) || '' },
         ...(paramStr(task)
@@ -280,7 +280,7 @@ const DetailSide = (): JSX.Element => {
       ],
     });
   } else if (task?.type === 'Action') {
-    taskParam = MetaKV({
+    taskInput = MetaKV({
       data: [
         { key: 'type', content: (task && typeStr(task)) || '' },
         {
@@ -289,6 +289,10 @@ const DetailSide = (): JSX.Element => {
         },
       ],
     });
+  } else if (task?.type === 'Log') {
+    taskInput = task.param?.content ? (
+      <pre className="log-content">{task.param.content}</pre>
+    ) : null;
   }
 
   let outputDataContent = null;
@@ -467,9 +471,9 @@ const DetailSide = (): JSX.Element => {
       {/* Meta */}
       <PanelTitle title="Task Meta" />
       {metaKVElement}
-      {/* Param  */}
-      <PanelTitle title="Param" />
-      {taskParam}
+      {/* Input: Param/Content  */}
+      <PanelTitle title={task?.type === 'Log' ? 'Content' : 'Param'} />
+      {taskInput}
       {/* Response */}
       <PanelTitle title={task?.subType === 'Locate' ? 'Element' : 'Output'} />
       <div className="item-list item-list-space-up">{outputDataContent}</div>

apps/report/src/utils.ts

@@ -253,6 +253,32 @@ The `deepThink` feature is a powerful feature that allows Midscene to call AI mo
 
 :::
 
+## Log Screenshot
+
+Log the current screenshot with a description in the report file.
+
+* Type
+
+```typescript
+function logScreenshot(title?: string, options?: Object): Promise<void>;
+```
+
+* Parameters:
+  * `title?: string` - Optional, the title of the screenshot, if not provided, the title will be 'untitled'.
+  * `options?: Object` - Optional, a configuration object containing:
+    * `content?: string` - The description of the screenshot.
+
+* Return Value:
+  * Returns a `Promise<void>`
+
+* Examples:
+
+```typescript
+await agent.logScreenshot('Login page', {
+  content: 'User A',
+});
+```
+
 ## Data Extraction
 
 ### `agent.aiQuery()`

apps/site/docs/en/API.mdx

@@ -145,7 +145,7 @@ web:
     # boolean, continue on network idle error, true for default
     continueOnNetworkIdleError: <boolean>
 
-  # string, the path to save the aiQuery result, optional
+  # string, the path to save the aiQuery/aiAssert result, optional
   output: <path-to-output-file>
 
   # boolean, if limit the popup to the current page, true for default in yaml script
@@ -232,6 +232,10 @@ tasks:
         deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element
         cacheable: <boolean> # optional, whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
+      # log the current screenshot with a description in the report file
+      - logScreenshot: <title> # optional, the title of the screenshot, if not provided, the title will be 'untitled'
+        content: <content> # optional, the screenshot description
+
       # Data Extraction
       # ----------------
 

apps/site/docs/en/automate-with-scripts-in-yaml.mdx

@@ -251,6 +251,35 @@ await agent.aiRightClick('页面顶部的文件名称', { deepThink: true });
 
 :::
 
+## 记录截图
+
+### `agent.logScreenshot()`
+
+在报告文件中记录当前截图，并添加描述。
+
+* 类型
+
+```typescript
+function logScreenshot(title?: string, options?: Object): Promise<void>;
+```
+
+* 参数：
+  * `title?: string` - 可选，截图的标题，如果未提供，则标题为 'untitled'。
+  * `options?: Object` - 可选，一个配置对象，包含：
+    * `content?: string` - 截图的描述。
+
+
+* 返回值：
+  * `Promise<void>`
+
+* 示例：
+
+```typescript
+await agent.logScreenshot('登录页面', {
+  content: '用户 A',
+});
+```
+
 ## 数据提取
 
 ### `agent.aiQuery()`

apps/site/docs/zh/API.mdx

@@ -145,7 +145,7 @@ web:
     # 是否在等待超时后继续，可选，默认 true
     continueOnNetworkIdleError: <boolean>
 
-  # 输出 aiQuery 结果的 JSON 文件路径，可选
+  # 输出 aiQuery/aiAssert 结果的 JSON 文件路径，可选
   output: <path-to-output-file>
 
   # 是否限制页面在当前 tab 打开，可选，默认 true
@@ -232,6 +232,10 @@ tasks:
         deepThink: <boolean> # 可选，是否使用深度思考（deepThink）来精确定位元素
         cacheable: <boolean> # 可选，当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
 
+      # 在报告文件中记录当前截图，并添加描述
+      - logScreenshot: <title> # 可选，截图的标题，如果未提供，则标题为 'untitled'
+        content: <content> # 可选，截图的描述
+
       # 数据提取
       # ----------------
 

apps/site/docs/zh/automate-with-scripts-in-yaml.mdx

@@ -361,7 +361,12 @@ export interface ExecutionRecorderItem {
   timing?: string;
 }
 
-export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion';
+export type ExecutionTaskType =
+  | 'Planning'
+  | 'Insight'
+  | 'Action'
+  | 'Assertion'
+  | 'Log';
 
 export interface ExecutorContext {
   task: ExecutionTask;
@@ -506,6 +511,18 @@ export type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<
 
 export type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;
 
+/*
+task - Log
+*/
+
+export type ExecutionTaskLogApply<
+  LogParam = {
+    content: string;
+  },
+> = ExecutionTaskApply<'Log', LogParam, void, void>;
+
+export type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;
+
 /*
 task - planning
 */

packages/core/src/types.ts

@@ -162,6 +162,11 @@ export interface MidsceneYamlFlowItemSleep {
   sleep: number;
 }
 
+export interface MidsceneYamlFlowItemLogScreenshot {
+  logScreenshot?: string; // optional, the title of the screenshot
+  content?: string;
+}
+
 export type MidsceneYamlFlowItem =
   | MidsceneYamlFlowItemAIAction
   | MidsceneYamlFlowItemAIAssert
@@ -173,7 +178,8 @@ export type MidsceneYamlFlowItem =
   | MidsceneYamlFlowItemAIInput
   | MidsceneYamlFlowItemAIKeyboardPress
   | MidsceneYamlFlowItemAIScroll
-  | MidsceneYamlFlowItemSleep;
+  | MidsceneYamlFlowItemSleep
+  | MidsceneYamlFlowItemLogScreenshot;
 
 export interface FreeFn {
   name: string;

packages/core/src/yaml.ts

@@ -42,3 +42,5 @@ export {
   getPlaceholderForType,
   blankResult,
 } from './component/playground/playground-utils';
+
+export { timeStr, filterBase64Value } from './utils';