define appium export in package.json & encapsulated screenshot picture path generation (#95)

* feat: define appium export in package.json

* feat: encapsulated screenshot picture path generation

Author: quanru

packages/midscene/src/utils.ts

@@ -145,8 +145,6 @@ export async function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-export const commonScreenshotParam = { type: 'jpeg', quality: 75 } as any;
-
 export function replacerForPageObject(key: string, value: any) {
   if (value && value.constructor?.name === 'Page') {
     return '[Page object]';

packages/web-integration/README.md

@@ -17,7 +17,7 @@ then execute the command to start appium server:
 appium --use-plugins=universal-xml
 ```
 
-now you can use run tests for the iOS/Android device:
+now you can use run tests for iOS/Android devices:
 
 ```bash
 npm run test:ai -- appium

packages/web-integration/package.json

@@ -27,6 +27,11 @@
       "import": "./dist/es/playwright-report.js",
       "require": "./dist/lib/playwright-report.js"
     },
+    "./appium": {
+      "types": "./dist/types/appium.d.ts",
+      "import": "./dist/es/appium.js",
+      "require": "./dist/lib/appium.js"
+    },
     "./debug": {
       "types": "./dist/types/debug.d.ts",
       "import": "./dist/es/debug.js",

packages/web-integration/src/appium/page.ts

@@ -1,10 +1,11 @@
 import fs from 'node:fs';
+import { getTmpFile } from '@midscene/core/utils';
 import { resizeImg } from '@midscene/shared/img';
 import { DOMParser } from '@xmldom/xmldom';
 import type { KeyInput as PuppeteerKeyInput } from 'puppeteer';
 import type { Browser } from 'webdriverio';
 import { type ElementInfo, clientExtractTextWithPosition } from '../extractor';
-import type { AbstractPage, MouseButton, screenshotOptions } from '../page';
+import type { AbstractPage, MouseButton } from '../page';
 
 type WebKeyInput = PuppeteerKeyInput;
 
@@ -37,21 +38,17 @@ export class Page implements AbstractPage {
     return infos;
   }
 
-  async screenshot(options: screenshotOptions): Promise<void> {
-    if (!options.path) {
-      throw new Error('path is required for screenshot');
-    }
-
+  async screenshot(): Promise<string> {
     const { width, height } = await this.browser.getWindowSize();
-    const screenshotBuffer = await this.browser.saveScreenshot(options.path);
+    const path = getTmpFile('png');
+    const screenshotBuffer = await this.browser.saveScreenshot(path);
     const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, {
       width,
       height,
     });
+    fs.writeFileSync(path, resizedScreenshotBuffer);
 
-    if (options.path) {
-      fs.writeFileSync(options.path, resizedScreenshotBuffer);
-    }
+    return path;
   }
 
   get mouse() {

packages/web-integration/src/common/tasks.ts

@@ -24,7 +24,7 @@ import Insight, {
   type PlanningActionParamWaitFor,
   type PlanningActionParamError,
 } from '@midscene/core';
-import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils';
+import { sleep } from '@midscene/core/utils';
 import { base64Encoded } from '@midscene/shared/img';
 import type { KeyInput } from 'puppeteer';
 import type { ElementInfo } from '../extractor';
@@ -56,15 +56,11 @@ export class PageTaskExecutor {
   }
 
   private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {
-    const file = getTmpFile('png');
-    await this.page.screenshot({
-      ...commonScreenshotParam,
-      path: file,
-    });
+    const file = await this.page.screenshot();
     const item: ExecutionRecorderItem = {
       type: 'screenshot',
       ts: Date.now(),
-      screenshot: base64Encoded(file),
+      screenshot: base64Encoded(file as string),
       timing,
     };
     return item;

packages/web-integration/src/common/utils.ts

@@ -23,9 +23,7 @@ export async function parseContextFromWebPage(
   assert(page, 'page is required');
 
   const url = page.url();
-  const file = getTmpFile('jpeg');
-  await page.screenshot({ path: file });
-
+  const file = await page.screenshot();
   const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
   const captureElementSnapshot = await page.getElementInfos();

packages/web-integration/src/debug/index.ts

@@ -3,7 +3,6 @@ import path from 'node:path';
 import type { WebPage } from '@/common/page';
 import type { ElementInfo } from '@/extractor';
 import { NodeType } from '@/extractor/constants';
-import { getTmpFile } from '@midscene/core/utils';
 import {
   processImageElementInfo,
   resizeImg,
@@ -21,8 +20,7 @@ export async function generateExtractData(
     disableSnapshot: boolean;
   },
 ) {
-  const file = getTmpFile('png');
-  await page.screenshot({ path: file });
+  const file = await page.screenshot();
   const screenshotBuffer = readFileSync(file);
 
   const inputImgBase64 = screenshotBuffer.toString('base64');

packages/web-integration/src/index.ts

@@ -3,6 +3,6 @@ export type { PlayWrightAiFixtureType } from './playwright';
 
 export { PuppeteerAgent } from './puppeteer';
 export { PlaywrightAgent } from './playwright';
-export { AppiumAgent } from './appium';
+export { AppiumAgent, AppiumPage } from './appium';
 
 export { generateExtractData } from './debug';

packages/web-integration/src/page.ts

@@ -1,20 +1,11 @@
 import type { WebKeyInput } from './common/page';
 import type { ElementInfo } from './extractor';
 
-type imageType = 'jpeg' | 'png';
-type encodingType = 'base64' | 'binary';
-
-export type screenshotOptions = {
-  path?: string;
-  encoding?: encodingType;
-  type?: imageType;
-  quality?: number;
-};
 export type MouseButton = 'left' | 'right' | 'middle';
 
 export abstract class AbstractPage {
   abstract pageType: string;
-  abstract screenshot(options?: screenshotOptions): Promise<void>;
+  abstract screenshot(): Promise<string>;
   abstract getElementInfos(): Promise<ElementInfo[]>;
   abstract url(): string;
 

packages/web-integration/src/puppeteer/base-page.ts

@@ -1,11 +1,12 @@
 import { readFileSync, writeFileSync } from 'node:fs';
+import { getTmpFile } from '@midscene/core/utils';
 import { resizeImg } from '@midscene/shared/img';
 import type { Page as PlaywrightPage } from 'playwright';
 import type { Page as PuppeteerPage } from 'puppeteer';
 import type { WebKeyInput } from '../common/page';
 import { getExtraReturnLogic } from '../common/utils';
 import type { ElementInfo } from '../extractor';
-import type { AbstractPage, screenshotOptions } from '../page';
+import type { AbstractPage } from '../page';
 import type { MouseButton } from '../page';
 
 export class Page<
@@ -37,12 +38,7 @@ export class Page<
     return captureElementSnapshot as ElementInfo[];
   }
 
-  async screenshot(options: screenshotOptions): Promise<void> {
-    const { path } = options;
-    if (!path) {
-      throw new Error('path is required for screenshot');
-    }
-
+  async screenshot(): Promise<string> {
     // get viewport size from page
     const viewportSize: {
       width: number;
@@ -56,6 +52,8 @@ export class Page<
       };
     });
 
+    const path = getTmpFile('jpeg');
+
     await this.page.screenshot({
       path,
       type: 'jpeg',
@@ -71,18 +69,7 @@ export class Page<
       writeFileSync(path, buf);
     }
 
-    // return await this.page.screenshot({
-    //   path,
-    //   type: 'jpeg',
-    //   quality: 75,
-    //   clip: {
-    //     x: 0,
-    //     y: 0,
-    //     width: viewportSize.width,
-    //     height: viewportSize.height,
-    //     scale: 1 / viewportSize.deviceScaleFactor,
-    //   },
-    // });
+    return path;
   }
 
   url(): string {

packages/web-integration/tests/ai/native/appium/dongchedi.test.ts

@@ -25,6 +25,9 @@ const IOS_OPTIONS = {
 
 const ANDROID_OPTIONS = {
   port: 4723,
+  waitforTimeout: 10000,
+  connectionRetryTimeout: 120000,
+  connectionRetryCount: 3,
   capabilities: {
     platformName: 'Android',
     'appium:automationName': 'UiAutomator2',
@@ -39,13 +42,16 @@ describe(
   'appium integration',
   () => {
     it('懂车帝查找小米 SU7', async () => {
-      const page = await launchPage(IOS_OPTIONS);
+      const page = await launchPage(ANDROID_OPTIONS);
       const mid = new AppiumAgent(page);
       await mid.aiAction('点击同意按钮');
+      await sleep(3000);
+      await mid.aiAction('点击允许获取应用位置信息');
+      await sleep(3000);
       await mid.aiAction('点击顶部输入框');
-      // await generateExtractData(page, './tmp');
-      await mid.aiAction('在输入框里输入"小米SU7"，并点击搜索');
-      // await sleep(3000);
+      await sleep(3000);
+      await mid.aiAction('在输入框里输入"SU7"，并点击搜索');
+      await sleep(3000);
       const items = await mid.aiQuery(
         '"{carName: string, price: number }[], return item name, price',
       );

packages/web-integration/tests/ai/web/playwright/tool.ts

@@ -1,4 +1,4 @@
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import type { WebPage } from '@/common/page';
 import { getElementInfos } from '@/debug';
@@ -16,10 +16,8 @@ export async function generateExtractData(
     disableSnapshot: boolean;
   },
 ) {
-  const buffer = await page.screenshot({
-    encoding: 'base64',
-  });
-  const inputImgBase64 = buffer.toString('base64');
+  const filePath = await page.screenshot();
+  const inputImgBase64 = readFileSync(filePath).toString('base64');
 
   const {
     elementsPositionInfo,

packages/web-integration/tests/unit-test/web-extractor.test.ts

@@ -1,7 +1,6 @@
 import path, { join } from 'node:path';
 import { parseContextFromWebPage } from '@/common/utils';
 import { generateExtractData } from '@/debug';
-import { getTmpFile } from '@midscene/core/utils';
 import { imageInfo } from '@midscene/shared/img';
 import { describe, expect, it } from 'vitest';
 import { launchPage } from '../ai/web/puppeteer/utils';
@@ -51,8 +50,7 @@ describe(
         },
       });
 
-      const shotpath = getTmpFile('jpeg');
-      await page.screenshot({ path: shotpath });
+      const shotpath = await page.screenshot();
 
       const info = await imageInfo(shotpath);
       expect(info.width).toBe(1080);
@@ -69,8 +67,7 @@ describe(
         },
       });
 
-      const shotpath = getTmpFile('jpeg');
-      await page.screenshot({ path: shotpath });
+      const shotpath = await page.screenshot();
 
       const info = await imageInfo(shotpath);
       expect(info.width).toBe(1080); // always 1x for screenshot