fix(web-integration): handle serialized static screenshots

packages/web-integration/src/static/static-page.ts

@@ -12,7 +12,38 @@ const ThrowNotImplemented = (methodName: string) => {
   );
 };
 
-type StaticPageUIContext = Omit<UIContext, 'deprecatedDpr'>;
+type SerializedStaticScreenshot = {
+  base64?: unknown;
+  _base64?: unknown;
+  type?: unknown;
+};
+
+type StaticPageUIContext = Omit<
+  UIContext,
+  'deprecatedDpr' | 'screenshot'
+> & {
+  screenshot: UIContext['screenshot'] | SerializedStaticScreenshot;
+};
+
+function screenshotBase64FromContext(
+  screenshot: StaticPageUIContext['screenshot'],
+): string {
+  const record = screenshot as SerializedStaticScreenshot;
+  const base64 = record.base64 ?? record._base64;
+  if (typeof base64 === 'string') {
+    return base64;
+  }
+
+  if (record.type === 'midscene_screenshot_ref') {
+    throw new Error(
+      'StaticPage screenshot is a serialized reference without base64 data',
+    );
+  }
+
+  throw new Error(
+    'StaticPage screenshot must include base64 data before execution',
+  );
+}
 
 export default class StaticPage implements AbstractInterface {
   interfaceType = 'static';
@@ -78,11 +109,7 @@ export default class StaticPage implements AbstractInterface {
   }
 
   async screenshotBase64() {
-    const screenshot = this.uiContext.screenshot;
-    if (typeof screenshot === 'object' && 'base64' in screenshot) {
-      return (screenshot as { base64: string }).base64;
-    }
-    return screenshot as unknown as string;
+    return screenshotBase64FromContext(this.uiContext.screenshot);
   }
 
   async url() {

packages/web-integration/tests/ai/web/static/static-page.test.ts

@@ -9,7 +9,7 @@ const dumpFilePath = join(__dirname, '../../fixtures/ui-context.json');
 const context = readFileSync(dumpFilePath, { encoding: 'utf-8' });
 const contextJson = JSON.parse(context);
 
-contextJson.screenshot = contextJson.screenshotBase64;
+contextJson.screenshot = { base64: contextJson.screenshotBase64 };
 
 describe(
   'static page agent',

packages/web-integration/tests/unit-test/playground-server.test.ts

@@ -1,7 +1,7 @@
 import { ScreenshotItem } from '@midscene/core';
 import { PlaygroundServer } from '@midscene/playground';
-import { StaticPage, StaticPageAgent } from '@midscene/web/static';
 import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { StaticPage, StaticPageAgent } from '../../src/static';
 
 describe('Playground Server', () => {
   let server: PlaygroundServer;
@@ -44,4 +44,32 @@ describe('Playground Server', () => {
     expect(context).toBeDefined();
     expect(context.context).toBe(contextValue);
   });
+
+  it('updates static context with a JSON-serialized ScreenshotItem', async () => {
+    const screenshotBase64 = 'data:image/png;base64,abc123';
+    const serializedScreenshot = JSON.parse(
+      JSON.stringify(ScreenshotItem.create(screenshotBase64, Date.now())),
+    );
+
+    const res = await fetch(`${serverBase}/action-space`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        context: {
+          shotSize: { width: 800, height: 600 },
+          shrunkShotToLogicalRatio: 1,
+          screenshot: serializedScreenshot,
+        },
+      }),
+    });
+
+    expect(res.status).toBe(200);
+
+    const screenshotRes = await fetch(`${serverBase}/screenshot`);
+    expect(screenshotRes.status).toBe(200);
+    const screenshot = await screenshotRes.json();
+    expect(screenshot.screenshot).toBe(screenshotBase64);
+  });
 });

packages/web-integration/tests/unit-test/static-page.test.ts

@@ -0,0 +1,56 @@
+import { ScreenshotItem } from '@midscene/core';
+import { describe, expect, it } from 'vitest';
+import { StaticPage } from '../../src/static';
+
+const screenshotBase64 = 'data:image/png;base64,abc123';
+
+function createContext(
+  screenshot: ConstructorParameters<typeof StaticPage>[0]['screenshot'],
+): ConstructorParameters<typeof StaticPage>[0] {
+  return {
+    shotSize: { width: 800, height: 600 },
+    shrunkShotToLogicalRatio: 1,
+    screenshot,
+  };
+}
+
+describe('StaticPage', () => {
+  it('returns base64 from a ScreenshotItem instance', async () => {
+    const page = new StaticPage(
+      createContext(ScreenshotItem.create(screenshotBase64, Date.now())),
+    );
+
+    await expect(page.screenshotBase64()).resolves.toBe(screenshotBase64);
+  });
+
+  it('returns base64 from a restored report screenshot object', async () => {
+    const page = new StaticPage(createContext({ base64: screenshotBase64 }));
+
+    await expect(page.screenshotBase64()).resolves.toBe(screenshotBase64);
+  });
+
+  it('returns base64 from a JSON-serialized ScreenshotItem', async () => {
+    const serializedScreenshot = JSON.parse(
+      JSON.stringify(ScreenshotItem.create(screenshotBase64, Date.now())),
+    );
+    const page = new StaticPage(createContext(serializedScreenshot));
+
+    await expect(page.screenshotBase64()).resolves.toBe(screenshotBase64);
+  });
+
+  it('rejects screenshot refs that do not include base64 data', async () => {
+    const page = new StaticPage(
+      createContext({
+        type: 'midscene_screenshot_ref',
+        id: 'screenshot-id',
+        capturedAt: Date.now(),
+        mimeType: 'image/png',
+        storage: 'inline',
+      }),
+    );
+
+    await expect(page.screenshotBase64()).rejects.toThrow(
+      'serialized reference without base64 data',
+    );
+  });
+});