feat(web-integration): support user expected xpath option for locate methods (#844)

* feat(web-integration): support user expected xpath option for locate methods

* docs(site): xpath option for api

* fix(ci): nx config

* docs(site): update docs

Author: quanru

apps/report/src/components/detail-side.tsx

@@ -198,14 +198,14 @@ const DetailSide = (): JSX.Element => {
             },
           ]
         : []),
-      {
-        key: 'cache',
-        content: task?.cache ? (
-          <pre>{JSON.stringify(task?.cache, undefined, 2)}</pre>
-        ) : (
-          'false'
-        ),
-      },
+      ...(task?.hitBy
+        ? [
+            {
+              key: 'hitBy',
+              content: <pre>{JSON.stringify(task?.hitBy, undefined, 2)}</pre>,
+            },
+          ]
+        : []),
       ...(task?.locate
         ? [
             {

apps/report/src/components/sidebar.tsx

@@ -18,7 +18,7 @@ const SideItem = (props: {
   const selectedClass = selected ? 'selected' : '';
   let statusText: JSX.Element | string = task.status;
 
-  const cacheEl = task.cache?.hit ? <span>(cache) </span> : null;
+  const cacheEl = task.hitBy?.from === 'Cache' ? <span>(cache) </span> : null;
 
   const deepThinkEl = (task as ExecutionTaskInsightLocate)?.log?.dump
     ?.deepThink ? (

apps/site/docs/en/API.mdx

@@ -108,7 +108,8 @@ function aiTap(locate: string, options?: Object): Promise<void>;
 
   - `locate: string` - A natural language description of the element to tap.
   - `options?: Object` - Optional, a configuration object containing:
-    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element.
+    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element. False by default.
+    - `xpath?: string` - The xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
     - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
 - Return Value:
@@ -140,7 +141,8 @@ function aiHover(locate: string, options?: Object): Promise<void>;
 
   - `locate: string` - A natural language description of the element to hover over.
   - `options?: Object` - Optional, a configuration object containing:
-    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element.
+    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element. False by default.
+    - `xpath?: string` - The xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
     - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
 - Return Value:
@@ -168,7 +170,8 @@ function aiInput(text: string, locate: string, options?: Object): Promise<void>;
   - `text: string` - The final text content that should be placed in the input element. Use blank string to clear the input.
   - `locate: string` - A natural language description of the element to input text into.
   - `options?: Object` - Optional, a configuration object containing:
-    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element.
+    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element. False by default.
+    - `xpath?: string` - The xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
     - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
     - `autoDismissKeyboard?: boolean` - If true, the keyboard will be dismissed after input text, only available in Android. (Default: true)
 
@@ -201,7 +204,8 @@ function aiKeyboardPress(
   - `key: string` - The web key to press, e.g. 'Enter', 'Tab', 'Escape', etc. Key Combination is not supported.
   - `locate?: string` - Optional, a natural language description of the element to press the key on.
   - `options?: Object` - Optional, a configuration object containing:
-    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element.
+    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element. False by default.
+    - `xpath?: string` - The xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
     - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
 - Return Value:
@@ -236,7 +240,8 @@ function aiScroll(
     - `distance: number` - Optional, the distance to scroll in px.
   - `locate?: string` - Optional, a natural language description of the element to scroll on. If not provided, Midscene will perform scroll on the current mouse position.
   - `options?: Object` - Optional, a configuration object containing:
-    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element.
+    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element. False by default.
+    - `xpath?: string` - The xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
     - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
 - Return Value:
@@ -266,7 +271,8 @@ function aiRightClick(locate: string, options?: Object): Promise<void>;
 
   - `locate: string` - A natural language description of the element to right-click on.
   - `options?: Object` - Optional, a configuration object containing:
-    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element.
+    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element. False by default.
+    - `xpath?: string` - The xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
     - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
 - Return Value:
@@ -286,7 +292,7 @@ await agent.aiRightClick('The file name at the top of the page', {
 
 :::tip About the `deepThink` feature
 
-The `deepThink` feature is a powerful feature that allows Midscene to call AI model twice to precisely locate the element. It is useful when the AI model find it hard to distinguish the element from its surroundings.
+The `deepThink` feature is a powerful feature that allows Midscene to call AI model twice to precisely locate the element. False by default. It is useful when the AI model find it hard to distinguish the element from its surroundings.
 
 :::
 
@@ -531,7 +537,8 @@ function aiLocate(
 
   - `locate: string` - A natural language description of the element to locate.
   - `options?: Object` - Optional, a configuration object containing:
-    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element.
+    - `deepThink?: boolean` - If true, Midscene will call AI model twice to precisely locate the element. False by default.
+    - `xpath?: string` - The xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
     - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
 - Return Value:

apps/site/docs/en/automate-with-scripts-in-yaml.mdx

@@ -206,24 +206,32 @@ tasks:
 
       # tap an element located by prompt
       - aiTap: <prompt>
-        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element
+        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element. False by default.
+        xpath: <xpath> # optional, the xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
+    - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
         cacheable: <boolean> # optional, whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
       # hover an element located by prompt
       - aiHover: <prompt>
-        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element
+        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element. False by default.
+        xpath: <xpath> # optional, the xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
+    - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
         cacheable: <boolean> # optional, whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
       # input text into an element located by prompt
       - aiInput: <final text content of the input>
         locate: <prompt>
-        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element
+        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element. False by default.
+        xpath: <xpath> # optional, the xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
+    - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
         cacheable: <boolean> # optional, whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
       # press a key (like Enter, Tab, Escape, etc.) on an element located by prompt
       - aiKeyboardPress: <key>
         locate: <prompt>
-        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element
+        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element. False by default.
+        xpath: <xpath> # optional, the xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
+    - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
         cacheable: <boolean> # optional, whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
       # scroll globally or on an element located by prompt
@@ -232,7 +240,9 @@ tasks:
         scrollType: 'once' # or 'untilTop' | 'untilBottom' | 'untilLeft' | 'untilRight'
         distance: <number> # optional, distance to scroll in px
         locate: <prompt> # optional, the element to scroll on
-        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element
+        deepThink: <boolean> # optional, whether to use deepThink to precisely locate the element. False by default.
+        xpath: <xpath> # optional, the xpath of the element to operate. If provided, Midscene will first use this xpath to locate the element before using the cache and the AI model. Empty by default.
+    - `cacheable?: boolean` - Whether cacheable when enabling [caching feature](./caching.mdx). True by default.
         cacheable: <boolean> # optional, whether cacheable when enabling [caching feature](./caching.mdx). True by default.
 
       # log the current screenshot with a description in the report file

apps/site/docs/zh/API.mdx

@@ -105,7 +105,8 @@ function aiTap(locate: string, options?: Object): Promise<void>;
 - 参数：
   - `locate: string` - 用自然语言描述的元素定位。
   - `options?: Object` - 可选，一个配置对象，包含：
-    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。
+    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。默认值为 False
+    - `xpath?: string` - 目标元素的 xpath 路径，用于执行当前操作。如果提供了这个 xpath，Midscene 会优先使用该 xpath 来找到元素，然后依次使用缓存和 AI 模型。默认值为空
     - `cacheable?: boolean` - 当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
 - 返回值：
 
@@ -133,7 +134,8 @@ function aiHover(locate: string, options?: Object): Promise<void>;
 - 参数：
   - `locate: string` - 用自然语言描述的元素定位。
   - `options?: Object` - 可选，一个配置对象，包含：
-    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。
+    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。默认值为 False
+    - `xpath?: string` - 目标元素的 xpath 路径，用于执行当前操作。如果提供了这个 xpath，Midscene 会优先使用该 xpath 来找到元素，然后依次使用缓存和 AI 模型。默认值为空
     - `cacheable?: boolean` - 当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
 - 返回值：
 
@@ -160,7 +162,8 @@ function aiInput(text: string, locate: string, options?: Object): Promise<void>;
   - `text: string` - 要输入的文本内容。使用空字符串可以清空输入框。
   - `locate: string` - 用自然语言描述的元素定位。
   - `options?: Object` - 可选，一个配置对象，包含：
-    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。
+    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。默认值为 False
+    - `xpath?: string` - 目标元素的 xpath 路径，用于执行当前操作。如果提供了这个 xpath，Midscene 会优先使用该 xpath 来找到元素，然后依次使用缓存和 AI 模型。默认值为空
     - `cacheable?: boolean` - 当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
     - `autoDismissKeyboard?: boolean` - 如果为 true，则键盘会在输入文本后自动关闭，仅在 Android 中有效。默认值为 true。
 
@@ -193,7 +196,8 @@ function aiKeyboardPress(
   - `key: string` - 要按下的键，如 `Enter`、`Tab`、`Escape` 等。不支持组合键。
   - `locate?: string` - 用自然语言描述的元素定位。
   - `options?: Object` - 可选，一个配置对象，包含：
-    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。
+    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。默认值为 False
+    - `xpath?: string` - 目标元素的 xpath 路径，用于执行当前操作。如果提供了这个 xpath，Midscene 会优先使用该 xpath 来找到元素，然后依次使用缓存和 AI 模型。默认值为空
     - `cacheable?: boolean` - 当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
 
 - 返回值：
@@ -228,7 +232,8 @@ function aiScroll(
     - `distance: number` - 滚动距离，单位为像素。
   - `locate?: string` - 用自然语言描述的元素定位。如果未传入，Midscene 会在当前鼠标位置滚动。
   - `options?: Object` - 可选，一个配置对象，包含：
-    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。
+    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。默认值为 False
+    - `xpath?: string` - 目标元素的 xpath 路径，用于执行当前操作。如果提供了这个 xpath，Midscene 会优先使用该 xpath 来找到元素，然后依次使用缓存和 AI 模型。默认值为空
     - `cacheable?: boolean` - 当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
 
 - 返回值：
@@ -258,7 +263,8 @@ function aiRightClick(locate: string, options?: Object): Promise<void>;
 
   - `locate: string` - 用自然语言描述的元素定位。
   - `options?: Object` - 可选，一个配置对象，包含：
-    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。
+    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。默认值为 False
+    - `xpath?: string` - 目标元素的 xpath 路径，用于执行当前操作。如果提供了这个 xpath，Midscene 会优先使用该 xpath 来找到元素，然后依次使用缓存和 AI 模型。默认值为空
     - `cacheable?: boolean` - 当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
 
 - 返回值：
@@ -529,7 +535,8 @@ function aiLocate(
 
   - `locate: string` - 用自然语言描述的元素定位。
   - `options?: Object` - 可选，一个配置对象，包含：
-    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。
+    - `deepThink?: boolean` - 是否开启深度思考。如果为 true，Midscene 会调用 AI 模型两次以精确定位元素。默认值为 False
+    - `xpath?: string` - 目标元素的 xpath 路径，用于执行当前操作。如果提供了这个 xpath，Midscene 会优先使用该 xpath 来找到元素，然后依次使用缓存和 AI 模型。默认值为空
     - `cacheable?: boolean` - 当启用 [缓存功能](./caching.mdx) 时，是否允许缓存当前 API 调用结果。默认值为 True
 
 - 返回值：