diff --git a/docs/channels/qa-channel.md b/docs/channels/qa-channel.md index c62ef42dabd..1ed0f3618b4 100644 --- a/docs/channels/qa-channel.md +++ b/docs/channels/qa-channel.md @@ -75,10 +75,13 @@ self-check, and writes a Markdown report under `.artifacts/qa-e2e/`. Private debugger UI: ```bash -pnpm qa:lab:build -pnpm openclaw qa ui +pnpm qa:lab:up ``` +That one command builds the QA site, starts the Docker-backed gateway + QA Lab +stack, and prints the QA Lab URL. From that site you can pick scenarios, choose +the model lane, launch individual runs, and watch results live. + Full repo-backed QA suite: ```bash @@ -96,10 +99,10 @@ Current scope is intentionally narrow: - threaded routing grammar - channel-owned message actions - Markdown reporting +- Docker-backed QA site with run controls Follow-up work will add: -- Dockerized OpenClaw orchestration - provider/model matrix execution - richer scenario discovery - OpenClaw-native orchestration later diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index f60aba5a27e..23642cb3532 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -21,13 +21,21 @@ Current pieces: - `qa/`: repo-backed seed assets for the kickoff task and baseline QA scenarios. -The long-term goal is a two-pane QA site: +The current QA operator flow is a two-pane QA site: - Left: Gateway dashboard (Control UI) with the agent. - Right: QA Lab, showing the Slack-ish transcript and scenario plan. -That lets an operator or automation loop give the agent a QA mission, observe -real channel behavior, and record what worked, failed, or stayed blocked. +Run it with: + +```bash +pnpm qa:lab:up +``` + +That builds the QA site, starts the Docker-backed gateway lane, and exposes the +QA Lab page where an operator or automation loop can give the agent a QA +mission, observe real channel behavior, and record what worked, failed, or +stayed blocked. ## Repo-backed seeds diff --git a/docs/help/testing.md b/docs/help/testing.md index 8d456b49986..36cac632cae 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -26,6 +26,7 @@ Most days: - Faster local full-suite run on a roomy machine: `pnpm test:max` - Direct Vitest watch loop: `pnpm test:watch` - Direct file targeting now routes extension/channel paths too: `pnpm test extensions/discord/src/monitor/message-handler.preflight.test.ts` +- Docker-backed QA site: `pnpm qa:lab:up` When you touch tests or want extra confidence: diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index a8c0eec6354..14043d81e48 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -1,5 +1,6 @@ import path from "node:path"; import { buildQaDockerHarnessImage, writeQaDockerHarnessFiles } from "./docker-harness.js"; +import { runQaDockerUp } from "./docker-up.runtime.js"; import { startQaLabServer } from "./lab-server.js"; import { startQaMockOpenAiServer } from "./mock-openai-server.js"; import { runQaSuite } from "./suite.js"; @@ -107,6 +108,31 @@ export async function runQaDockerBuildImageCommand(opts: { image?: string }) { process.stdout.write(`QA docker image: ${result.imageName}\n`); } +export async function runQaDockerUpCommand(opts: { + outputDir?: string; + gatewayPort?: number; + qaLabPort?: number; + providerBaseUrl?: string; + image?: string; + usePrebuiltImage?: boolean; + skipUiBuild?: boolean; +}) { + const result = await runQaDockerUp({ + repoRoot: process.cwd(), + outputDir: opts.outputDir ? path.resolve(opts.outputDir) : undefined, + gatewayPort: Number.isFinite(opts.gatewayPort) ? opts.gatewayPort : undefined, + qaLabPort: Number.isFinite(opts.qaLabPort) ? opts.qaLabPort : undefined, + providerBaseUrl: opts.providerBaseUrl, + image: opts.image, + usePrebuiltImage: opts.usePrebuiltImage, + skipUiBuild: opts.skipUiBuild, + }); + process.stdout.write(`QA docker dir: ${result.outputDir}\n`); + process.stdout.write(`QA Lab UI: ${result.qaLabUrl}\n`); + process.stdout.write(`Gateway UI: ${result.gatewayUrl}\n`); + process.stdout.write(`Stop: ${result.stopCommand}\n`); +} + export async function runQaMockOpenAiCommand(opts: { host?: string; port?: number }) { const server = await startQaMockOpenAiServer({ host: opts.host, diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index d83dd9cfc7f..48421fe8c8b 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -57,6 +57,19 @@ async function runQaDockerBuildImage(opts: { image?: string }) { await runtime.runQaDockerBuildImageCommand(opts); } +async function runQaDockerUp(opts: { + outputDir?: string; + gatewayPort?: number; + qaLabPort?: number; + providerBaseUrl?: string; + image?: string; + usePrebuiltImage?: boolean; + skipUiBuild?: boolean; +}) { + const runtime = await loadQaLabCliRuntime(); + await runtime.runQaDockerUpCommand(opts); +} + async function runQaMockOpenAi(opts: { host?: string; port?: number }) { const runtime = await loadQaLabCliRuntime(); await runtime.runQaMockOpenAiCommand(opts); @@ -165,6 +178,29 @@ export function registerQaLabCli(program: Command) { await runQaDockerBuildImage(opts); }); + qa.command("up") + .description("Build the QA site, start the Docker-backed QA stack, and print the QA Lab URL") + .option("--output-dir ", "Output directory for docker-compose + state files") + .option("--gateway-port ", "Gateway host port", (value: string) => Number(value)) + .option("--qa-lab-port ", "QA lab host port", (value: string) => Number(value)) + .option("--provider-base-url ", "Provider base URL for the QA gateway") + .option("--image ", "Image tag", "openclaw:qa-local-prebaked") + .option("--use-prebuilt-image", "Use image: instead of build: in docker-compose", false) + .option("--skip-ui-build", "Skip pnpm qa:lab:build before starting Docker", false) + .action( + async (opts: { + outputDir?: string; + gatewayPort?: number; + qaLabPort?: number; + providerBaseUrl?: string; + image?: string; + usePrebuiltImage?: boolean; + skipUiBuild?: boolean; + }) => { + await runQaDockerUp(opts); + }, + ); + qa.command("mock-openai") .description("Run the local mock OpenAI Responses API server for QA") .option("--host ", "Bind host", "127.0.0.1") diff --git a/extensions/qa-lab/src/docker-up.runtime.test.ts b/extensions/qa-lab/src/docker-up.runtime.test.ts new file mode 100644 index 00000000000..79cac537289 --- /dev/null +++ b/extensions/qa-lab/src/docker-up.runtime.test.ts @@ -0,0 +1,85 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { runQaDockerUp } from "./docker-up.runtime.js"; + +describe("runQaDockerUp", () => { + it("builds the QA UI, writes the harness, starts compose, and waits for health", async () => { + const calls: string[] = []; + const fetchCalls: string[] = []; + const responseQueue = [false, true, false, true]; + const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-")); + + try { + const result = await runQaDockerUp( + { + repoRoot: "/repo/openclaw", + outputDir, + gatewayPort: 18889, + qaLabPort: 43124, + }, + { + async runCommand(command, args, cwd) { + calls.push([command, ...args, `@${cwd}`].join(" ")); + return { stdout: "", stderr: "" }; + }, + fetchImpl: vi.fn(async (input: string) => { + fetchCalls.push(input); + return { ok: responseQueue.shift() ?? true }; + }), + sleepImpl: vi.fn(async () => {}), + }, + ); + + expect(calls).toEqual([ + "pnpm qa:lab:build @/repo/openclaw", + expect.stringContaining( + `docker compose -f ${outputDir}/docker-compose.qa.yml up --build -d @/repo/openclaw`, + ), + ]); + expect(fetchCalls).toEqual([ + "http://127.0.0.1:43124/healthz", + "http://127.0.0.1:43124/healthz", + "http://127.0.0.1:18889/healthz", + "http://127.0.0.1:18889/healthz", + ]); + expect(result.qaLabUrl).toBe("http://127.0.0.1:43124"); + expect(result.gatewayUrl).toBe("http://127.0.0.1:18889/"); + expect(result.composeFile).toBe(`${outputDir}/docker-compose.qa.yml`); + expect(result.stopCommand).toBe(`docker compose -f ${outputDir}/docker-compose.qa.yml down`); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + }); + + it("skips UI build and compose --build for prebuilt images", async () => { + const calls: string[] = []; + const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-")); + + try { + await runQaDockerUp( + { + repoRoot: "/repo/openclaw", + outputDir, + usePrebuiltImage: true, + skipUiBuild: true, + }, + { + async runCommand(command, args, cwd) { + calls.push([command, ...args, `@${cwd}`].join(" ")); + return { stdout: "", stderr: "" }; + }, + fetchImpl: vi.fn(async () => ({ ok: true })), + sleepImpl: vi.fn(async () => {}), + }, + ); + + expect(calls).toEqual([ + `docker compose -f ${outputDir}/docker-compose.qa.yml up -d @/repo/openclaw`, + ]); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + }); +}); diff --git a/extensions/qa-lab/src/docker-up.runtime.ts b/extensions/qa-lab/src/docker-up.runtime.ts new file mode 100644 index 00000000000..d5978a4e65a --- /dev/null +++ b/extensions/qa-lab/src/docker-up.runtime.ts @@ -0,0 +1,148 @@ +import { execFile } from "node:child_process"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { writeQaDockerHarnessFiles } from "./docker-harness.js"; + +type QaDockerUpResult = { + outputDir: string; + composeFile: string; + qaLabUrl: string; + gatewayUrl: string; + stopCommand: string; +}; + +type RunCommand = ( + command: string, + args: string[], + cwd: string, +) => Promise<{ stdout: string; stderr: string }>; + +type FetchLike = (input: string) => Promise<{ ok: boolean }>; + +const DEFAULT_QA_DOCKER_DIR = path.resolve(process.cwd(), ".artifacts/qa-docker"); + +function describeError(error: unknown) { + if (error instanceof Error) { + return error.message; + } + if (typeof error === "string") { + return error; + } + return JSON.stringify(error); +} + +async function execCommand(command: string, args: string[], cwd: string) { + return await new Promise<{ stdout: string; stderr: string }>((resolve, reject) => { + execFile(command, args, { cwd, encoding: "utf8" }, (error, stdout, stderr) => { + if (error) { + reject( + new Error( + stderr.trim() || stdout.trim() || `Command failed: ${[command, ...args].join(" ")}`, + ), + ); + return; + } + resolve({ stdout, stderr }); + }); + }); +} + +async function waitForHealth( + url: string, + deps: { + fetchImpl: FetchLike; + sleepImpl: (ms: number) => Promise; + timeoutMs?: number; + pollMs?: number; + }, +) { + const timeoutMs = deps.timeoutMs ?? 120_000; + const pollMs = deps.pollMs ?? 1_000; + const deadline = Date.now() + timeoutMs; + let lastError: unknown = null; + + while (Date.now() < deadline) { + try { + const response = await deps.fetchImpl(url); + if (response.ok) { + return; + } + lastError = new Error(`Health check returned non-OK for ${url}`); + } catch (error) { + lastError = error; + } + await deps.sleepImpl(pollMs); + } + + throw new Error( + `Timed out waiting for ${url}${lastError ? `: ${describeError(lastError)}` : ""}`, + ); +} + +export async function runQaDockerUp( + params: { + repoRoot?: string; + outputDir?: string; + gatewayPort?: number; + qaLabPort?: number; + providerBaseUrl?: string; + image?: string; + usePrebuiltImage?: boolean; + skipUiBuild?: boolean; + }, + deps?: { + runCommand?: RunCommand; + fetchImpl?: FetchLike; + sleepImpl?: (ms: number) => Promise; + }, +): Promise { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const outputDir = path.resolve(params.outputDir ?? DEFAULT_QA_DOCKER_DIR); + const gatewayPort = params.gatewayPort ?? 18789; + const qaLabPort = params.qaLabPort ?? 43124; + const runCommand = deps?.runCommand ?? execCommand; + const fetchImpl = + deps?.fetchImpl ?? + (async (input: string) => { + return await fetch(input); + }); + const sleepImpl = deps?.sleepImpl ?? sleep; + + if (!params.skipUiBuild) { + await runCommand("pnpm", ["qa:lab:build"], repoRoot); + } + + await writeQaDockerHarnessFiles({ + outputDir, + repoRoot, + gatewayPort, + qaLabPort, + providerBaseUrl: params.providerBaseUrl, + imageName: params.image, + usePrebuiltImage: params.usePrebuiltImage, + includeQaLabUi: true, + }); + + const composeFile = path.join(outputDir, "docker-compose.qa.yml"); + const composeArgs = ["compose", "-f", composeFile, "up"]; + if (!params.usePrebuiltImage) { + composeArgs.push("--build"); + } + composeArgs.push("-d"); + + await runCommand("docker", composeArgs, repoRoot); + + const qaLabUrl = `http://127.0.0.1:${qaLabPort}`; + const gatewayUrl = `http://127.0.0.1:${gatewayPort}/`; + + await waitForHealth(`${qaLabUrl}/healthz`, { fetchImpl, sleepImpl }); + await waitForHealth(`${gatewayUrl}healthz`, { fetchImpl, sleepImpl }); + + return { + outputDir, + composeFile, + qaLabUrl, + gatewayUrl, + stopCommand: `docker compose -f ${composeFile} down`, + }; +} diff --git a/package.json b/package.json index 8626d2fe714..0de5bdc1d06 100644 --- a/package.json +++ b/package.json @@ -1115,6 +1115,7 @@ "qa:e2e": "node --import tsx scripts/qa-e2e.ts", "qa:lab:build": "vite build --config extensions/qa-lab/web/vite.config.ts", "qa:lab:ui": "pnpm openclaw qa ui", + "qa:lab:up": "node --import tsx scripts/qa-lab-up.ts", "release:check": "pnpm check:base-config-schema && pnpm check:bundled-channel-config-metadata && pnpm config:docs:check && pnpm plugin-sdk:check-exports && pnpm plugin-sdk:api:check && node --import tsx scripts/release-check.ts", "release:openclaw:npm:check": "node --import tsx scripts/openclaw-npm-release-check.ts", "release:openclaw:npm:verify-published": "node --import tsx scripts/openclaw-npm-postpublish-verify.ts", diff --git a/scripts/qa-lab-up.ts b/scripts/qa-lab-up.ts new file mode 100644 index 00000000000..1061000a090 --- /dev/null +++ b/scripts/qa-lab-up.ts @@ -0,0 +1,53 @@ +import { parseArgs } from "node:util"; +import { runQaDockerUpCommand } from "../extensions/qa-lab/src/cli.runtime.ts"; + +const { values } = parseArgs({ + options: { + help: { type: "boolean", short: "h" }, + "output-dir": { type: "string" }, + "gateway-port": { type: "string" }, + "qa-lab-port": { type: "string" }, + "provider-base-url": { type: "string" }, + image: { type: "string" }, + "use-prebuilt-image": { type: "boolean" }, + "skip-ui-build": { type: "boolean" }, + }, + allowPositionals: false, +}); + +if (values.help) { + process.stdout.write(`Usage: pnpm qa:lab:up [options] + +Options: + --output-dir + --gateway-port + --qa-lab-port + --provider-base-url + --image + --use-prebuilt-image + --skip-ui-build + -h, --help +`); + process.exit(0); +} + +const parsePort = (value: string | undefined) => { + if (!value) { + return undefined; + } + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid port: ${value}`); + } + return parsed; +}; + +await runQaDockerUpCommand({ + outputDir: values["output-dir"], + gatewayPort: parsePort(values["gateway-port"]), + qaLabPort: parsePort(values["qa-lab-port"]), + providerBaseUrl: values["provider-base-url"], + image: values.image, + usePrebuiltImage: values["use-prebuilt-image"], + skipUiBuild: values["skip-ui-build"], +});