mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-28 20:46:57 +02:00
1519 lines
51 KiB
JavaScript
1519 lines
51 KiB
JavaScript
import { spawn } from "node:child_process";
|
|
import fs from "node:fs";
|
|
import { mkdir, readFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const DEFAULT_E2E_IMAGE = "openclaw-docker-e2e:local";
|
|
const DEFAULT_PARALLELISM = 10;
|
|
const DEFAULT_TAIL_PARALLELISM = 10;
|
|
const DEFAULT_FAILURE_TAIL_LINES = 80;
|
|
const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000;
|
|
const DEFAULT_LANE_START_STAGGER_MS = 2_000;
|
|
const DEFAULT_LIVE_RETRIES = 1;
|
|
const DEFAULT_STATUS_INTERVAL_MS = 30_000;
|
|
const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000;
|
|
const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
|
|
const DEFAULT_PROFILE = "all";
|
|
const RELEASE_PATH_PROFILE = "release-path";
|
|
const LIVE_PROFILE_TIMEOUT_MS = 20 * 60 * 1000;
|
|
const LIVE_CLI_TIMEOUT_MS = 20 * 60 * 1000;
|
|
const LIVE_ACP_TIMEOUT_MS = 20 * 60 * 1000;
|
|
const OPENWEBUI_TIMEOUT_MS = 20 * 60 * 1000;
|
|
const BUNDLED_UPDATE_TIMEOUT_MS = 20 * 60 * 1000;
|
|
const DEFAULT_RESOURCE_LIMITS = {
|
|
docker: DEFAULT_PARALLELISM,
|
|
live: 9,
|
|
"live:claude": 4,
|
|
"live:codex": 4,
|
|
"live:droid": 4,
|
|
"live:gemini": 4,
|
|
"live:opencode": 4,
|
|
npm: 10,
|
|
service: 7,
|
|
};
|
|
const LIVE_RETRY_PATTERNS = [
|
|
/529\b/i,
|
|
/overloaded/i,
|
|
/capacity/i,
|
|
/rate.?limit/i,
|
|
/gateway closed \(1000 normal closure\)/i,
|
|
/ECONNRESET|ETIMEDOUT|ENOTFOUND/i,
|
|
];
|
|
|
|
const bundledChannelLaneCommand =
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps";
|
|
|
|
function lane(name, command, options = {}) {
|
|
return {
|
|
cacheKey: options.cacheKey,
|
|
command,
|
|
estimateSeconds: options.estimateSeconds,
|
|
live: options.live === true,
|
|
name,
|
|
retryPatterns: options.retryPatterns ?? [],
|
|
retries: options.retries ?? 0,
|
|
resources: options.resources ?? [],
|
|
timeoutMs: options.timeoutMs,
|
|
weight: options.weight ?? 1,
|
|
};
|
|
}
|
|
|
|
function liveProviderResource(provider) {
|
|
if (!provider) {
|
|
return undefined;
|
|
}
|
|
if (provider === "claude-cli" || provider === "claude") {
|
|
return "live:claude";
|
|
}
|
|
if (provider === "codex-cli" || provider === "codex") {
|
|
return "live:codex";
|
|
}
|
|
if (provider === "droid") {
|
|
return "live:droid";
|
|
}
|
|
if (provider === "google-gemini-cli" || provider === "gemini") {
|
|
return "live:gemini";
|
|
}
|
|
if (provider === "opencode") {
|
|
return "live:opencode";
|
|
}
|
|
if (provider === "openai") {
|
|
return "live:openai";
|
|
}
|
|
return `live:${provider}`;
|
|
}
|
|
|
|
function liveProviderResources(options) {
|
|
const providers = options.providers ?? (options.provider ? [options.provider] : []);
|
|
return providers.map(liveProviderResource).filter(Boolean);
|
|
}
|
|
|
|
function liveLane(name, command, options = {}) {
|
|
return lane(name, command, {
|
|
...options,
|
|
live: true,
|
|
resources: ["live", ...liveProviderResources(options), ...(options.resources ?? [])],
|
|
retryPatterns: options.retryPatterns ?? LIVE_RETRY_PATTERNS,
|
|
retries: options.retries ?? DEFAULT_LIVE_RETRIES,
|
|
weight: options.weight ?? 3,
|
|
});
|
|
}
|
|
|
|
function npmLane(name, command, options = {}) {
|
|
return lane(name, command, {
|
|
...options,
|
|
resources: ["npm", ...(options.resources ?? [])],
|
|
weight: options.weight ?? 2,
|
|
});
|
|
}
|
|
|
|
function serviceLane(name, command, options = {}) {
|
|
return lane(name, command, {
|
|
...options,
|
|
resources: ["service", ...(options.resources ?? [])],
|
|
weight: options.weight ?? 2,
|
|
});
|
|
}
|
|
|
|
const bundledScenarioLanes = [
|
|
npmLane(
|
|
"bundled-channel-telegram",
|
|
`OPENCLAW_BUNDLED_CHANNELS=telegram ${bundledChannelLaneCommand}`,
|
|
),
|
|
npmLane(
|
|
"bundled-channel-discord",
|
|
`OPENCLAW_BUNDLED_CHANNELS=discord ${bundledChannelLaneCommand}`,
|
|
),
|
|
npmLane("bundled-channel-slack", `OPENCLAW_BUNDLED_CHANNELS=slack ${bundledChannelLaneCommand}`),
|
|
npmLane(
|
|
"bundled-channel-feishu",
|
|
`OPENCLAW_BUNDLED_CHANNELS=feishu ${bundledChannelLaneCommand}`,
|
|
),
|
|
npmLane(
|
|
"bundled-channel-memory-lancedb",
|
|
`OPENCLAW_BUNDLED_CHANNELS=memory-lancedb ${bundledChannelLaneCommand}`,
|
|
),
|
|
npmLane(
|
|
"bundled-channel-update-telegram",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=telegram OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
|
),
|
|
npmLane(
|
|
"bundled-channel-update-discord",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=discord OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
|
),
|
|
npmLane(
|
|
"bundled-channel-update-slack",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=slack OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
|
),
|
|
npmLane(
|
|
"bundled-channel-update-feishu",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=feishu OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
|
),
|
|
npmLane(
|
|
"bundled-channel-update-memory-lancedb",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=memory-lancedb OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
|
),
|
|
npmLane(
|
|
"bundled-channel-update-acpx",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=acpx OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
{ timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS },
|
|
),
|
|
npmLane(
|
|
"bundled-channel-root-owned",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
),
|
|
npmLane(
|
|
"bundled-channel-setup-entry",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
),
|
|
npmLane(
|
|
"bundled-channel-load-failure",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps",
|
|
),
|
|
npmLane(
|
|
"bundled-channel-disabled-config",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=1 pnpm test:docker:bundled-channel-deps",
|
|
),
|
|
];
|
|
|
|
const lanes = [
|
|
liveLane("live-models", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-models", {
|
|
providers: ["claude-cli", "codex-cli", "google-gemini-cli"],
|
|
timeoutMs: LIVE_PROFILE_TIMEOUT_MS,
|
|
weight: 4,
|
|
}),
|
|
liveLane("live-gateway", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-gateway", {
|
|
providers: ["claude-cli", "codex-cli", "google-gemini-cli"],
|
|
timeoutMs: LIVE_PROFILE_TIMEOUT_MS,
|
|
weight: 4,
|
|
}),
|
|
liveLane(
|
|
"live-cli-backend-claude",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:claude",
|
|
{
|
|
cacheKey: "cli-backend-claude",
|
|
provider: "claude-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_CLI_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
liveLane(
|
|
"live-cli-backend-gemini",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:gemini",
|
|
{
|
|
cacheKey: "cli-backend-gemini",
|
|
provider: "google-gemini-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_CLI_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
serviceLane("openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui", {
|
|
timeoutMs: OPENWEBUI_TIMEOUT_MS,
|
|
weight: 5,
|
|
}),
|
|
serviceLane("onboard", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:onboard", {
|
|
weight: 2,
|
|
}),
|
|
npmLane(
|
|
"npm-onboard-channel-agent",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent",
|
|
{ resources: ["service"], weight: 3 },
|
|
),
|
|
serviceLane("gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"),
|
|
serviceLane(
|
|
"agents-delete-shared-workspace",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:agents-delete-shared-workspace",
|
|
),
|
|
serviceLane("mcp-channels", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:mcp-channels", {
|
|
resources: ["npm"],
|
|
weight: 3,
|
|
}),
|
|
lane("pi-bundle-mcp-tools", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:pi-bundle-mcp-tools"),
|
|
lane("crestodian-rescue", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:crestodian-rescue"),
|
|
lane("crestodian-planner", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:crestodian-planner"),
|
|
serviceLane(
|
|
"cron-mcp-cleanup",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:cron-mcp-cleanup",
|
|
{ resources: ["npm"], weight: 3 },
|
|
),
|
|
npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", {
|
|
weight: 3,
|
|
}),
|
|
npmLane(
|
|
"update-channel-switch",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-channel-switch",
|
|
{
|
|
timeoutMs: 30 * 60 * 1000,
|
|
weight: 3,
|
|
},
|
|
),
|
|
lane("plugins", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins", {
|
|
resources: ["npm", "service"],
|
|
weight: 6,
|
|
}),
|
|
npmLane("plugin-update", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugin-update"),
|
|
serviceLane("config-reload", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:config-reload"),
|
|
...bundledScenarioLanes,
|
|
lane("openai-image-auth", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-image-auth"),
|
|
lane(
|
|
"crestodian-first-run",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:crestodian-first-run",
|
|
),
|
|
lane(
|
|
"session-runtime-context",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:session-runtime-context",
|
|
),
|
|
lane("qr", "pnpm test:docker:qr"),
|
|
];
|
|
|
|
const exclusiveLanes = [
|
|
serviceLane(
|
|
"openai-web-search-minimal",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-web-search-minimal",
|
|
{ timeoutMs: 8 * 60 * 1000 },
|
|
),
|
|
liveLane(
|
|
"live-codex-harness",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-codex-harness",
|
|
{
|
|
cacheKey: "codex-harness",
|
|
provider: "codex-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
liveLane("live-codex-bind", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-codex-bind", {
|
|
cacheKey: "codex-harness",
|
|
provider: "codex-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
|
weight: 3,
|
|
}),
|
|
liveLane(
|
|
"live-cli-backend-codex",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:codex",
|
|
{
|
|
cacheKey: "cli-backend-codex",
|
|
provider: "codex-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_CLI_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
liveLane(
|
|
"live-acp-bind-claude",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:claude",
|
|
{
|
|
cacheKey: "acp-bind-claude",
|
|
provider: "claude-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
liveLane(
|
|
"live-acp-bind-codex",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:codex",
|
|
{
|
|
cacheKey: "acp-bind-codex",
|
|
provider: "codex-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
liveLane(
|
|
"live-acp-bind-droid",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:droid",
|
|
{
|
|
cacheKey: "acp-bind-droid",
|
|
provider: "droid",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
liveLane(
|
|
"live-acp-bind-gemini",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:gemini",
|
|
{
|
|
cacheKey: "acp-bind-gemini",
|
|
provider: "google-gemini-cli",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
liveLane(
|
|
"live-acp-bind-opencode",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:opencode",
|
|
{
|
|
cacheKey: "acp-bind-opencode",
|
|
provider: "opencode",
|
|
resources: ["npm"],
|
|
timeoutMs: LIVE_ACP_TIMEOUT_MS,
|
|
weight: 3,
|
|
},
|
|
),
|
|
];
|
|
|
|
const tailLanes = exclusiveLanes;
|
|
|
|
const releasePathChunks = {
|
|
core: [
|
|
lane("qr", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:qr"),
|
|
serviceLane("onboard", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:onboard", {
|
|
weight: 2,
|
|
}),
|
|
serviceLane("gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"),
|
|
serviceLane("config-reload", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:config-reload"),
|
|
lane(
|
|
"session-runtime-context",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:session-runtime-context",
|
|
),
|
|
lane(
|
|
"pi-bundle-mcp-tools",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:pi-bundle-mcp-tools",
|
|
),
|
|
serviceLane("mcp-channels", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:mcp-channels", {
|
|
resources: ["npm"],
|
|
weight: 3,
|
|
}),
|
|
],
|
|
"package-update": [
|
|
npmLane("install-e2e", "OPENCLAW_E2E_MODELS=both pnpm test:install:e2e", {
|
|
resources: ["service"],
|
|
weight: 4,
|
|
}),
|
|
npmLane(
|
|
"npm-onboard-channel-agent",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent",
|
|
{ resources: ["service"], weight: 3 },
|
|
),
|
|
npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", {
|
|
weight: 3,
|
|
}),
|
|
npmLane(
|
|
"update-channel-switch",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-channel-switch",
|
|
{
|
|
timeoutMs: 30 * 60 * 1000,
|
|
weight: 3,
|
|
},
|
|
),
|
|
],
|
|
"plugins-integrations": [
|
|
lane("plugins", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins", {
|
|
resources: ["npm", "service"],
|
|
weight: 6,
|
|
}),
|
|
npmLane("plugin-update", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugin-update"),
|
|
npmLane(
|
|
"bundled-channel-deps",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:bundled-channel-deps",
|
|
{ resources: ["service"], weight: 3 },
|
|
),
|
|
serviceLane(
|
|
"cron-mcp-cleanup",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:cron-mcp-cleanup",
|
|
{
|
|
resources: ["npm"],
|
|
weight: 3,
|
|
},
|
|
),
|
|
serviceLane(
|
|
"openai-web-search-minimal",
|
|
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-web-search-minimal",
|
|
{ timeoutMs: 8 * 60 * 1000 },
|
|
),
|
|
],
|
|
};
|
|
|
|
function releasePathChunkLanes(chunk, options = {}) {
|
|
const base = releasePathChunks[chunk];
|
|
if (!base) {
|
|
throw new Error(
|
|
`OPENCLAW_DOCKER_ALL_CHUNK must be one of: ${Object.keys(releasePathChunks).join(", ")}. Got: ${JSON.stringify(chunk)}`,
|
|
);
|
|
}
|
|
if (chunk !== "plugins-integrations" || !options.includeOpenWebUI) {
|
|
return base;
|
|
}
|
|
return [
|
|
...base,
|
|
serviceLane("openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui", {
|
|
timeoutMs: OPENWEBUI_TIMEOUT_MS,
|
|
weight: 5,
|
|
}),
|
|
];
|
|
}
|
|
|
|
function allReleasePathLanes(options = {}) {
|
|
return Object.keys(releasePathChunks).flatMap((chunk) =>
|
|
releasePathChunkLanes(chunk, {
|
|
includeOpenWebUI: chunk === "plugins-integrations" && options.includeOpenWebUI,
|
|
}),
|
|
);
|
|
}
|
|
|
|
function parseLaneSelection(raw) {
|
|
if (!raw) {
|
|
return [];
|
|
}
|
|
return [
|
|
...new Set(
|
|
String(raw)
|
|
.split(/[,\s]+/u)
|
|
.map((token) => token.trim())
|
|
.filter(Boolean),
|
|
),
|
|
];
|
|
}
|
|
|
|
function dedupeLanes(poolLanes) {
|
|
const byName = new Map();
|
|
for (const poolLane of poolLanes) {
|
|
if (!byName.has(poolLane.name)) {
|
|
byName.set(poolLane.name, poolLane);
|
|
}
|
|
}
|
|
return [...byName.values()];
|
|
}
|
|
|
|
function selectNamedLanes(poolLanes, selectedNames, label) {
|
|
const byName = new Map(poolLanes.map((poolLane) => [poolLane.name, poolLane]));
|
|
const missing = selectedNames.filter((name) => !byName.has(name));
|
|
if (missing.length > 0) {
|
|
throw new Error(
|
|
`${label} unknown lane(s): ${missing.join(", ")}. Available lanes: ${[...byName.keys()]
|
|
.toSorted((a, b) => a.localeCompare(b))
|
|
.join(", ")}`,
|
|
);
|
|
}
|
|
return selectedNames.map((name) => byName.get(name));
|
|
}
|
|
|
|
function parsePositiveInt(raw, fallback, label) {
|
|
if (!raw) {
|
|
return fallback;
|
|
}
|
|
const parsed = Number(raw);
|
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseNonNegativeInt(raw, fallback, label) {
|
|
if (!raw) {
|
|
return fallback;
|
|
}
|
|
const parsed = Number(raw);
|
|
if (!Number.isInteger(parsed) || parsed < 0) {
|
|
throw new Error(`${label} must be a non-negative integer. Got: ${JSON.stringify(raw)}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseBool(raw, fallback) {
|
|
if (raw === undefined || raw === "") {
|
|
return fallback;
|
|
}
|
|
return !/^(?:0|false|no)$/i.test(raw);
|
|
}
|
|
|
|
function parseLiveMode(raw) {
|
|
const mode = raw || "all";
|
|
if (mode === "all" || mode === "skip" || mode === "only") {
|
|
return mode;
|
|
}
|
|
throw new Error(
|
|
`OPENCLAW_DOCKER_ALL_LIVE_MODE must be one of: all, skip, only. Got: ${JSON.stringify(raw)}`,
|
|
);
|
|
}
|
|
|
|
function parseProfile(raw) {
|
|
const profile = raw || DEFAULT_PROFILE;
|
|
if (profile === DEFAULT_PROFILE || profile === RELEASE_PATH_PROFILE) {
|
|
return profile;
|
|
}
|
|
throw new Error(
|
|
`OPENCLAW_DOCKER_ALL_PROFILE must be one of: ${DEFAULT_PROFILE}, ${RELEASE_PATH_PROFILE}. Got: ${JSON.stringify(raw)}`,
|
|
);
|
|
}
|
|
|
|
function applyLiveMode(poolLanes, mode) {
|
|
if (mode === "all") {
|
|
return poolLanes;
|
|
}
|
|
return poolLanes.filter((poolLane) => (mode === "only" ? poolLane.live : !poolLane.live));
|
|
}
|
|
|
|
function applyLiveRetries(poolLanes, retries) {
|
|
return poolLanes.map((poolLane) => (poolLane.live ? { ...poolLane, retries } : poolLane));
|
|
}
|
|
|
|
function resourceLimitsSummary(resourceLimits) {
|
|
return Object.entries(resourceLimits)
|
|
.map(([resource, limit]) => `${resource}=${String(limit)}`)
|
|
.join(" ");
|
|
}
|
|
|
|
function resourceLimitEnvName(resource) {
|
|
return `OPENCLAW_DOCKER_ALL_${resource.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_LIMIT`;
|
|
}
|
|
|
|
function parseResourceLimit(env, resource, parallelism, fallback) {
|
|
const envName = resourceLimitEnvName(resource);
|
|
return parsePositiveInt(env[envName], Math.min(parallelism, fallback), envName);
|
|
}
|
|
|
|
function parseSchedulerOptions(env, parallelism) {
|
|
const weightLimit = parsePositiveInt(
|
|
env.OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT,
|
|
parallelism,
|
|
"OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT",
|
|
);
|
|
const resourceLimits = {};
|
|
for (const [resource, fallback] of Object.entries(DEFAULT_RESOURCE_LIMITS)) {
|
|
resourceLimits[resource] = parseResourceLimit(env, resource, parallelism, fallback);
|
|
}
|
|
return {
|
|
resourceLimits,
|
|
weightLimit,
|
|
};
|
|
}
|
|
|
|
function laneWeight(poolLane) {
|
|
return Math.max(1, poolLane.weight ?? 1);
|
|
}
|
|
|
|
function laneResources(poolLane) {
|
|
return ["docker", ...(poolLane.resources ?? [])];
|
|
}
|
|
|
|
function laneSummary(poolLane) {
|
|
const resources = laneResources(poolLane).join(",");
|
|
const timeout = poolLane.timeoutMs ? ` timeout=${Math.round(poolLane.timeoutMs / 1000)}s` : "";
|
|
const retries = poolLane.retries > 0 ? ` retries=${poolLane.retries}` : "";
|
|
const cache = poolLane.cacheKey ? ` cache=${poolLane.cacheKey}` : "";
|
|
return `${poolLane.name}(w=${laneWeight(poolLane)} r=${resources}${timeout}${retries}${cache})`;
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => {
|
|
setTimeout(resolve, ms);
|
|
});
|
|
}
|
|
|
|
function utcStampForPath() {
|
|
return new Date().toISOString().replaceAll("-", "").replaceAll(":", "").replace(/\..*$/, "Z");
|
|
}
|
|
|
|
function utcStamp() {
|
|
return new Date().toISOString().replace(/\..*$/, "Z");
|
|
}
|
|
|
|
function appendExtension(env, extension) {
|
|
const current = env.OPENCLAW_DOCKER_BUILD_EXTENSIONS ?? env.OPENCLAW_EXTENSIONS ?? "";
|
|
const tokens = current.split(/\s+/).filter(Boolean);
|
|
if (!tokens.includes(extension)) {
|
|
tokens.push(extension);
|
|
}
|
|
env.OPENCLAW_DOCKER_BUILD_EXTENSIONS = tokens.join(" ");
|
|
}
|
|
|
|
function commandEnv(extra = {}) {
|
|
return {
|
|
...process.env,
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
function shellQuote(value) {
|
|
return `'${String(value).replaceAll("'", "'\\''")}'`;
|
|
}
|
|
|
|
function buildLaneRerunCommand(name, baseEnv) {
|
|
const build = name.startsWith("live-") ? "1" : "0";
|
|
const env = [
|
|
["OPENCLAW_DOCKER_ALL_LANES", name],
|
|
["OPENCLAW_DOCKER_ALL_BUILD", build],
|
|
["OPENCLAW_DOCKER_ALL_PREFLIGHT", "0"],
|
|
["OPENCLAW_SKIP_DOCKER_BUILD", "1"],
|
|
["OPENCLAW_DOCKER_E2E_IMAGE", baseEnv.OPENCLAW_DOCKER_E2E_IMAGE || DEFAULT_E2E_IMAGE],
|
|
];
|
|
return `${env.map(([key, value]) => `${key}=${shellQuote(value)}`).join(" ")} pnpm test:docker:all`;
|
|
}
|
|
|
|
function timingSeconds(timingStore, poolLane) {
|
|
const fromStore = timingStore?.lanes?.[poolLane.name]?.durationSeconds;
|
|
if (typeof fromStore === "number" && Number.isFinite(fromStore) && fromStore > 0) {
|
|
return fromStore;
|
|
}
|
|
return poolLane.estimateSeconds ?? 0;
|
|
}
|
|
|
|
function orderLanes(poolLanes, timingStore) {
|
|
return poolLanes
|
|
.map((poolLane, index) => ({ index, poolLane, seconds: timingSeconds(timingStore, poolLane) }))
|
|
.toSorted((a, b) => b.seconds - a.seconds || a.index - b.index)
|
|
.map(({ poolLane }) => poolLane);
|
|
}
|
|
|
|
async function loadTimingStore(file, enabled) {
|
|
if (!enabled) {
|
|
return { enabled: false, file, lanes: {}, version: 1 };
|
|
}
|
|
const raw = await readFile(file, "utf8").catch(() => "");
|
|
if (!raw.trim()) {
|
|
return { enabled: true, file, lanes: {}, version: 1 };
|
|
}
|
|
try {
|
|
const parsed = JSON.parse(raw);
|
|
return {
|
|
enabled: true,
|
|
file,
|
|
lanes: parsed && typeof parsed.lanes === "object" && parsed.lanes ? parsed.lanes : {},
|
|
version: 1,
|
|
};
|
|
} catch (error) {
|
|
console.warn(
|
|
`WARN: ignoring unreadable Docker lane timings ${file}: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
return { enabled: true, file, lanes: {}, version: 1 };
|
|
}
|
|
}
|
|
|
|
async function writeTimingStore(timingStore, results) {
|
|
if (!timingStore.enabled || results.length === 0) {
|
|
return;
|
|
}
|
|
const next = {
|
|
lanes: { ...timingStore.lanes },
|
|
updatedAt: new Date().toISOString(),
|
|
version: 1,
|
|
};
|
|
for (const result of results) {
|
|
if (!result || typeof result.elapsedSeconds !== "number") {
|
|
continue;
|
|
}
|
|
next.lanes[result.name] = {
|
|
durationSeconds: result.elapsedSeconds,
|
|
status: result.status,
|
|
timedOut: result.timedOut,
|
|
updatedAt: new Date().toISOString(),
|
|
};
|
|
}
|
|
await mkdir(path.dirname(timingStore.file), { recursive: true });
|
|
await fs.promises.writeFile(timingStore.file, `${JSON.stringify(next, null, 2)}\n`);
|
|
timingStore.lanes = next.lanes;
|
|
console.log(`==> Docker lane timings: ${timingStore.file}`);
|
|
}
|
|
|
|
async function writeRunSummary(logDir, summary) {
|
|
const file = path.join(logDir, "summary.json");
|
|
const payload = {
|
|
...summary,
|
|
finishedAt: new Date().toISOString(),
|
|
version: 1,
|
|
};
|
|
await fs.promises.writeFile(file, `${JSON.stringify(payload, null, 2)}\n`);
|
|
console.log(`==> Docker run summary: ${file}`);
|
|
}
|
|
|
|
function printLaneManifest(label, poolLanes, timingStore) {
|
|
console.log(`==> ${label} lanes (${poolLanes.length})`);
|
|
for (const [index, poolLane] of poolLanes.entries()) {
|
|
const seconds = timingSeconds(timingStore, poolLane);
|
|
const estimate = seconds > 0 ? ` last=${Math.round(seconds)}s` : "";
|
|
console.log(` ${index + 1}. ${laneSummary(poolLane)}${estimate}`);
|
|
}
|
|
}
|
|
|
|
function lanesNeedBundledPackage(poolLanes) {
|
|
return poolLanes.some(
|
|
(poolLane) =>
|
|
poolLane.name === "npm-onboard-channel-agent" || poolLane.name.startsWith("bundled-channel"),
|
|
);
|
|
}
|
|
|
|
function dockerPreflightContainerNames(raw) {
|
|
return raw
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim().split(/\s+/, 1)[0])
|
|
.filter((name) =>
|
|
/^(?:openclaw-(?:gateway-e2e|openwebui|openwebui-gateway|config-reload-e2e)-)/.test(name),
|
|
);
|
|
}
|
|
|
|
function runShellCommand({ command, env, label, logFile, timeoutMs }) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn("bash", ["-lc", command], {
|
|
cwd: ROOT_DIR,
|
|
detached: process.platform !== "win32",
|
|
env,
|
|
stdio: logFile ? ["ignore", "pipe", "pipe"] : "inherit",
|
|
});
|
|
activeChildren.add(child);
|
|
let timedOut = false;
|
|
let killTimer;
|
|
const timeoutTimer =
|
|
timeoutMs > 0
|
|
? setTimeout(() => {
|
|
timedOut = true;
|
|
if (stream) {
|
|
stream.write(`\n==> [${label}] timeout after ${timeoutMs}ms; sending SIGTERM\n`);
|
|
}
|
|
terminateChild(child, "SIGTERM");
|
|
killTimer = setTimeout(() => terminateChild(child, "SIGKILL"), 10_000);
|
|
killTimer.unref?.();
|
|
}, timeoutMs)
|
|
: undefined;
|
|
timeoutTimer?.unref?.();
|
|
|
|
let stream;
|
|
if (logFile) {
|
|
stream = fs.createWriteStream(logFile, { flags: "a" });
|
|
stream.write(`==> [${label}] command: ${command}\n`);
|
|
stream.write(`==> [${label}] started: ${utcStamp()}\n`);
|
|
child.stdout.pipe(stream, { end: false });
|
|
child.stderr.pipe(stream, { end: false });
|
|
}
|
|
|
|
child.on("close", (status, signal) => {
|
|
if (timeoutTimer) {
|
|
clearTimeout(timeoutTimer);
|
|
}
|
|
if (killTimer) {
|
|
clearTimeout(killTimer);
|
|
}
|
|
activeChildren.delete(child);
|
|
const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
|
|
if (stream) {
|
|
stream.write(`\n==> [${label}] finished: ${utcStamp()} status=${exitCode}\n`);
|
|
stream.end();
|
|
}
|
|
resolve({ signal, status: exitCode, timedOut });
|
|
});
|
|
});
|
|
}
|
|
|
|
function runShellCaptureCommand({ command, env, label, timeoutMs }) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn("bash", ["-lc", command], {
|
|
cwd: ROOT_DIR,
|
|
detached: process.platform !== "win32",
|
|
env,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
activeChildren.add(child);
|
|
let stdout = "";
|
|
let stderr = "";
|
|
let timedOut = false;
|
|
const timeoutTimer =
|
|
timeoutMs > 0
|
|
? setTimeout(() => {
|
|
timedOut = true;
|
|
terminateChild(child, "SIGTERM");
|
|
setTimeout(() => terminateChild(child, "SIGKILL"), 10_000).unref?.();
|
|
}, timeoutMs)
|
|
: undefined;
|
|
timeoutTimer?.unref?.();
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += String(chunk);
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += String(chunk);
|
|
});
|
|
child.on("close", (status, signal) => {
|
|
if (timeoutTimer) {
|
|
clearTimeout(timeoutTimer);
|
|
}
|
|
activeChildren.delete(child);
|
|
const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
|
|
resolve({ label, signal, status: exitCode, stderr, stdout, timedOut });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function runForeground(label, command, env) {
|
|
console.log(`==> ${label}`);
|
|
const result = await runShellCommand({ command, env, label });
|
|
if (result.status !== 0) {
|
|
throw new Error(`${label} failed with status ${result.status}`);
|
|
}
|
|
}
|
|
|
|
async function runForegroundGroup(entries, env) {
|
|
const results = await Promise.allSettled(
|
|
entries.map(async ([label, command]) => {
|
|
await runForeground(label, command, env);
|
|
}),
|
|
);
|
|
const failures = results
|
|
.map((result, index) => ({ result, entry: entries[index] }))
|
|
.filter(({ result }) => result.status === "rejected");
|
|
if (failures.length > 0) {
|
|
throw new Error(
|
|
failures
|
|
.map(
|
|
({ result, entry }) => `${entry[0]}: ${result.reason?.message ?? String(result.reason)}`,
|
|
)
|
|
.join("\n"),
|
|
);
|
|
}
|
|
}
|
|
|
|
async function runDockerPreflight(baseEnv, options) {
|
|
if (!options.enabled) {
|
|
console.log("==> Docker preflight: skipped");
|
|
return;
|
|
}
|
|
console.log("==> Docker preflight");
|
|
const version = await runShellCaptureCommand({
|
|
command: "docker version --format '{{.Server.Version}}'",
|
|
env: baseEnv,
|
|
label: "docker-version",
|
|
timeoutMs: 20_000,
|
|
});
|
|
if (version.status !== 0) {
|
|
throw new Error(
|
|
`Docker preflight failed: docker version status=${version.status}\n${version.stderr}${version.stdout}`,
|
|
);
|
|
}
|
|
console.log(`==> Docker server: ${version.stdout.trim()}`);
|
|
|
|
if (options.cleanup) {
|
|
const stale = await runShellCaptureCommand({
|
|
command:
|
|
"docker ps -a --filter status=created --filter status=exited --filter status=dead --format '{{.Names}} {{.Status}}'",
|
|
env: baseEnv,
|
|
label: "docker-stale-list",
|
|
timeoutMs: 20_000,
|
|
});
|
|
if (stale.status === 0) {
|
|
const names = dockerPreflightContainerNames(stale.stdout);
|
|
if (names.length > 0) {
|
|
console.log(`==> Docker preflight cleanup: ${names.join(", ")}`);
|
|
const cleanup = await runShellCommand({
|
|
command: `docker rm -f ${names.map(shellQuote).join(" ")}`,
|
|
env: baseEnv,
|
|
label: "docker-stale-cleanup",
|
|
timeoutMs: 90_000,
|
|
});
|
|
if (cleanup.status !== 0) {
|
|
throw new Error(`Docker preflight cleanup failed with status ${cleanup.status}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const startedAt = Date.now();
|
|
const run = await runShellCommand({
|
|
command: "docker run --rm alpine:3.20 true",
|
|
env: baseEnv,
|
|
label: "docker-run-smoke",
|
|
timeoutMs: options.runTimeoutMs,
|
|
});
|
|
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
|
|
if (run.status !== 0) {
|
|
throw new Error(
|
|
`Docker preflight failed: docker run alpine:3.20 true status=${run.status} elapsed=${elapsedSeconds}s`,
|
|
);
|
|
}
|
|
console.log(`==> Docker preflight run: ${elapsedSeconds}s`);
|
|
}
|
|
|
|
async function prepareBundledChannelPackage(baseEnv, logDir) {
|
|
if (baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ) {
|
|
console.log(`==> Bundled channel package: ${baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ}`);
|
|
return;
|
|
}
|
|
|
|
const packDir = path.join(logDir, "bundled-channel-package");
|
|
await mkdir(packDir, { recursive: true });
|
|
const packScript = [
|
|
"set -euo pipefail",
|
|
"node --import tsx --input-type=module -e \"const { writePackageDistInventory } = await import('./src/infra/package-dist-inventory.ts'); await writePackageDistInventory(process.cwd());\"",
|
|
"npm pack --silent --ignore-scripts --pack-destination /tmp/openclaw-pack >/tmp/openclaw-pack.out",
|
|
"cat /tmp/openclaw-pack.out",
|
|
].join("\n");
|
|
await runForeground(
|
|
"Pack bundled channel package once from Docker E2E image",
|
|
[
|
|
"docker run --rm",
|
|
"-e COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
|
|
`-v ${shellQuote(packDir)}:/tmp/openclaw-pack`,
|
|
shellQuote(baseEnv.OPENCLAW_DOCKER_E2E_IMAGE),
|
|
"bash -lc",
|
|
shellQuote(packScript),
|
|
].join(" "),
|
|
baseEnv,
|
|
);
|
|
|
|
const packed = (await fs.promises.readdir(packDir))
|
|
.filter((entry) => /^openclaw-.*\.tgz$/.test(entry))
|
|
.toSorted()
|
|
.at(-1);
|
|
if (!packed) {
|
|
throw new Error(`missing packed OpenClaw tarball in ${packDir}`);
|
|
}
|
|
baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ = path.join(packDir, packed);
|
|
baseEnv.OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD = "0";
|
|
baseEnv.OPENCLAW_NPM_ONBOARD_PACKAGE_TGZ = baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ;
|
|
baseEnv.OPENCLAW_NPM_ONBOARD_HOST_BUILD = "0";
|
|
console.log(`==> Bundled channel package: ${baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ}`);
|
|
}
|
|
|
|
function laneEnv(name, baseEnv, logDir, cacheKey) {
|
|
const env = {
|
|
...baseEnv,
|
|
};
|
|
const cacheName = cacheKey || name;
|
|
if (!process.env.OPENCLAW_DOCKER_CLI_TOOLS_DIR) {
|
|
env.OPENCLAW_DOCKER_CLI_TOOLS_DIR = path.join(logDir, `${cacheName}-cli-tools`);
|
|
}
|
|
if (!process.env.OPENCLAW_DOCKER_CACHE_HOME_DIR) {
|
|
env.OPENCLAW_DOCKER_CACHE_HOME_DIR = path.join(logDir, `${cacheName}-cache`);
|
|
}
|
|
return env;
|
|
}
|
|
|
|
async function runLane(lane, baseEnv, logDir, fallbackTimeoutMs) {
|
|
const { command, name } = lane;
|
|
const timeoutMs = lane.timeoutMs ?? fallbackTimeoutMs;
|
|
const logFile = path.join(logDir, `${name}.log`);
|
|
const env = laneEnv(name, baseEnv, logDir, lane.cacheKey);
|
|
await mkdir(env.OPENCLAW_DOCKER_CLI_TOOLS_DIR, { recursive: true });
|
|
await mkdir(env.OPENCLAW_DOCKER_CACHE_HOME_DIR, { recursive: true });
|
|
await fs.promises.writeFile(
|
|
logFile,
|
|
[
|
|
`==> [${name}] cli tools dir: ${env.OPENCLAW_DOCKER_CLI_TOOLS_DIR}`,
|
|
`==> [${name}] cache dir: ${env.OPENCLAW_DOCKER_CACHE_HOME_DIR}`,
|
|
`==> [${name}] timeout: ${timeoutMs}ms`,
|
|
`==> [${name}] retries: ${lane.retries ?? 0}`,
|
|
"",
|
|
].join("\n"),
|
|
);
|
|
console.log(`==> [${name}] start`);
|
|
const startedAt = Date.now();
|
|
let result;
|
|
const maxAttempts = 1 + Math.max(0, lane.retries ?? 0);
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
if (attempt > 1) {
|
|
await fs.promises.appendFile(logFile, `\n==> [${name}] retry attempt ${attempt}\n`);
|
|
console.log(`==> [${name}] retry ${attempt}/${maxAttempts}`);
|
|
}
|
|
result = await runShellCommand({ command, env, label: name, logFile, timeoutMs });
|
|
if (result.status === 0 || attempt >= maxAttempts) {
|
|
break;
|
|
}
|
|
const retryable =
|
|
result.timedOut || (await laneLogMatchesRetryPattern(logFile, lane.retryPatterns));
|
|
if (!retryable) {
|
|
break;
|
|
}
|
|
}
|
|
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
|
|
if (result.status === 0) {
|
|
console.log(`==> [${name}] pass ${elapsedSeconds}s`);
|
|
} else {
|
|
const timeoutLabel = result.timedOut ? " timeout" : "";
|
|
console.error(
|
|
`==> [${name}] fail${timeoutLabel} status=${result.status} ${elapsedSeconds}s log=${logFile}`,
|
|
);
|
|
}
|
|
return {
|
|
command,
|
|
logFile,
|
|
name,
|
|
elapsedSeconds,
|
|
rerunCommand: buildLaneRerunCommand(name, baseEnv),
|
|
status: result.status,
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
async function runLanePool(poolLanes, baseEnv, logDir, parallelism, options) {
|
|
const failures = [];
|
|
const results = [];
|
|
const pending = [...poolLanes];
|
|
const running = new Set();
|
|
const active = {
|
|
count: 0,
|
|
resources: new Map(),
|
|
weight: 0,
|
|
};
|
|
const activeLanes = new Map();
|
|
let lastLaneStartAt = 0;
|
|
let laneStartQueue = Promise.resolve();
|
|
const statusTimer =
|
|
options.statusIntervalMs > 0
|
|
? setInterval(() => {
|
|
const runningSummary = [...activeLanes.values()]
|
|
.map((entry) => `${entry.name}:${Math.round((Date.now() - entry.startedAt) / 1000)}s`)
|
|
.join(", ");
|
|
const resources = [...active.resources.entries()]
|
|
.map(([resource, value]) => `${resource}=${value}`)
|
|
.join(" ");
|
|
console.log(
|
|
`==> [${options.poolLabel}] active=${active.count} pending=${pending.length} ${resources}${
|
|
runningSummary ? ` lanes=${runningSummary}` : ""
|
|
}`,
|
|
);
|
|
}, options.statusIntervalMs)
|
|
: undefined;
|
|
statusTimer?.unref?.();
|
|
|
|
async function waitForLaneStartSlot() {
|
|
if (options.startStaggerMs <= 0) {
|
|
return;
|
|
}
|
|
const previous = laneStartQueue;
|
|
let releaseQueue;
|
|
laneStartQueue = new Promise((resolve) => {
|
|
releaseQueue = resolve;
|
|
});
|
|
await previous;
|
|
const waitMs = Math.max(0, lastLaneStartAt + options.startStaggerMs - Date.now());
|
|
if (waitMs > 0) {
|
|
await sleep(waitMs);
|
|
}
|
|
lastLaneStartAt = Date.now();
|
|
releaseQueue();
|
|
}
|
|
|
|
function canStartLane(candidate) {
|
|
const weight = laneWeight(candidate);
|
|
if (active.count >= parallelism || active.weight + weight > options.weightLimit) {
|
|
return false;
|
|
}
|
|
for (const resource of laneResources(candidate)) {
|
|
const limit = options.resourceLimits[resource] ?? options.weightLimit;
|
|
const current = active.resources.get(resource) ?? 0;
|
|
if (current + weight > limit) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function reserve(candidate) {
|
|
const weight = laneWeight(candidate);
|
|
active.count += 1;
|
|
active.weight += weight;
|
|
for (const resource of laneResources(candidate)) {
|
|
active.resources.set(resource, (active.resources.get(resource) ?? 0) + weight);
|
|
}
|
|
}
|
|
|
|
function release(candidate) {
|
|
const weight = laneWeight(candidate);
|
|
active.count -= 1;
|
|
active.weight -= weight;
|
|
for (const resource of laneResources(candidate)) {
|
|
const next = (active.resources.get(resource) ?? 0) - weight;
|
|
if (next > 0) {
|
|
active.resources.set(resource, next);
|
|
} else {
|
|
active.resources.delete(resource);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function startLane(poolLane) {
|
|
await waitForLaneStartSlot();
|
|
reserve(poolLane);
|
|
activeLanes.set(poolLane.name, { name: poolLane.name, startedAt: Date.now() });
|
|
let promise;
|
|
promise = runLane(poolLane, baseEnv, logDir, options.timeoutMs)
|
|
.then((result) => ({ lane: poolLane, promise, result }))
|
|
.finally(() => {
|
|
activeLanes.delete(poolLane.name);
|
|
release(poolLane);
|
|
});
|
|
running.add(promise);
|
|
}
|
|
|
|
try {
|
|
while (pending.length > 0 || running.size > 0) {
|
|
let started = false;
|
|
if (!options.failFast || failures.length === 0) {
|
|
for (let index = 0; index < pending.length; ) {
|
|
const candidate = pending[index];
|
|
if (!canStartLane(candidate)) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
pending.splice(index, 1);
|
|
await startLane(candidate);
|
|
started = true;
|
|
}
|
|
}
|
|
|
|
if (started) {
|
|
continue;
|
|
}
|
|
if (running.size === 0) {
|
|
const blocked = pending.map(laneSummary).join(", ");
|
|
throw new Error(`No Docker lanes fit scheduler limits: ${blocked}`);
|
|
}
|
|
|
|
const { promise, result } = await Promise.race(running);
|
|
running.delete(promise);
|
|
results.push(result);
|
|
if (result.status !== 0) {
|
|
failures.push(result);
|
|
}
|
|
if (options.failFast && failures.length > 0) {
|
|
const remainingResults = await Promise.all(running);
|
|
running.clear();
|
|
for (const remaining of remainingResults) {
|
|
results.push(remaining.result);
|
|
if (remaining.result.status !== 0) {
|
|
failures.push(remaining.result);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
} finally {
|
|
if (statusTimer) {
|
|
clearInterval(statusTimer);
|
|
}
|
|
}
|
|
|
|
return { failures, results };
|
|
}
|
|
|
|
async function tailFile(file, lines) {
|
|
const content = await readFile(file, "utf8").catch(() => "");
|
|
const tail = content.split(/\r?\n/).slice(-lines).join("\n");
|
|
return tail.trimEnd();
|
|
}
|
|
|
|
async function laneLogMatchesRetryPattern(logFile, patterns) {
|
|
if (!patterns || patterns.length === 0) {
|
|
return false;
|
|
}
|
|
const tail = await tailFile(logFile, 160);
|
|
return patterns.some((pattern) => pattern.test(tail));
|
|
}
|
|
|
|
async function printFailureSummary(failures, tailLines) {
|
|
console.error(`ERROR: ${failures.length} Docker lane(s) failed.`);
|
|
for (const failure of failures) {
|
|
console.error(`---- ${failure.name} failed (status=${failure.status}): ${failure.logFile}`);
|
|
const tail = await tailFile(failure.logFile, tailLines);
|
|
if (tail) {
|
|
console.error(tail);
|
|
}
|
|
}
|
|
}
|
|
|
|
const activeChildren = new Set();
|
|
function terminateChild(child, signal) {
|
|
if (process.platform !== "win32" && child.pid) {
|
|
try {
|
|
process.kill(-child.pid, signal);
|
|
return;
|
|
} catch {
|
|
// Fall back to killing the direct child below.
|
|
}
|
|
}
|
|
child.kill(signal);
|
|
}
|
|
|
|
function terminateActiveChildren(signal) {
|
|
for (const child of activeChildren) {
|
|
terminateChild(child, signal);
|
|
}
|
|
}
|
|
|
|
process.on("SIGINT", () => {
|
|
terminateActiveChildren("SIGINT");
|
|
process.exit(130);
|
|
});
|
|
process.on("SIGTERM", () => {
|
|
terminateActiveChildren("SIGTERM");
|
|
process.exit(143);
|
|
});
|
|
|
|
async function main() {
|
|
const runStartedAt = new Date().toISOString();
|
|
const parallelism = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_PARALLELISM,
|
|
DEFAULT_PARALLELISM,
|
|
"OPENCLAW_DOCKER_ALL_PARALLELISM",
|
|
);
|
|
const tailParallelism = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM,
|
|
Math.min(parallelism, DEFAULT_TAIL_PARALLELISM),
|
|
"OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM",
|
|
);
|
|
const tailLines = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES,
|
|
DEFAULT_FAILURE_TAIL_LINES,
|
|
"OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES",
|
|
);
|
|
const laneTimeoutMs = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS,
|
|
DEFAULT_LANE_TIMEOUT_MS,
|
|
"OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS",
|
|
);
|
|
const laneStartStaggerMs = parseNonNegativeInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_START_STAGGER_MS,
|
|
DEFAULT_LANE_START_STAGGER_MS,
|
|
"OPENCLAW_DOCKER_ALL_START_STAGGER_MS",
|
|
);
|
|
const statusIntervalMs = parseNonNegativeInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS,
|
|
DEFAULT_STATUS_INTERVAL_MS,
|
|
"OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS",
|
|
);
|
|
const preflightRunTimeoutMs = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS,
|
|
DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS,
|
|
"OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS",
|
|
);
|
|
const failFast = parseBool(process.env.OPENCLAW_DOCKER_ALL_FAIL_FAST, true);
|
|
const dryRun = parseBool(process.env.OPENCLAW_DOCKER_ALL_DRY_RUN, false);
|
|
const preflightEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT, true);
|
|
const preflightCleanup = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_CLEANUP, true);
|
|
const timingsEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_TIMINGS, true);
|
|
const buildEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_BUILD, true);
|
|
const profile = parseProfile(process.env.OPENCLAW_DOCKER_ALL_PROFILE);
|
|
const releaseChunk = process.env.OPENCLAW_DOCKER_ALL_CHUNK || process.env.DOCKER_E2E_CHUNK || "";
|
|
const includeOpenWebUI = parseBool(
|
|
process.env.OPENCLAW_DOCKER_ALL_INCLUDE_OPENWEBUI ?? process.env.INCLUDE_OPENWEBUI,
|
|
true,
|
|
);
|
|
const selectedLaneNamesRaw =
|
|
process.env.OPENCLAW_DOCKER_ALL_LANES || process.env.DOCKER_E2E_LANES || "";
|
|
const selectedLaneNames = parseLaneSelection(selectedLaneNamesRaw);
|
|
if (selectedLaneNamesRaw && selectedLaneNames.length === 0) {
|
|
throw new Error("OPENCLAW_DOCKER_ALL_LANES must include at least one lane name");
|
|
}
|
|
const liveMode = parseLiveMode(process.env.OPENCLAW_DOCKER_ALL_LIVE_MODE);
|
|
const liveRetries = parseNonNegativeInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_LIVE_RETRIES,
|
|
DEFAULT_LIVE_RETRIES,
|
|
"OPENCLAW_DOCKER_ALL_LIVE_RETRIES",
|
|
);
|
|
const timingsFile = path.resolve(
|
|
process.env.OPENCLAW_DOCKER_ALL_TIMINGS_FILE || DEFAULT_TIMINGS_FILE,
|
|
);
|
|
const runId = process.env.OPENCLAW_DOCKER_ALL_RUN_ID || utcStampForPath();
|
|
const logDir = path.resolve(
|
|
process.env.OPENCLAW_DOCKER_ALL_LOG_DIR ||
|
|
path.join(ROOT_DIR, ".artifacts/docker-tests", runId),
|
|
);
|
|
await mkdir(logDir, { recursive: true });
|
|
|
|
const baseEnv = commandEnv({
|
|
OPENCLAW_DOCKER_E2E_IMAGE: process.env.OPENCLAW_DOCKER_E2E_IMAGE || DEFAULT_E2E_IMAGE,
|
|
});
|
|
appendExtension(baseEnv, "matrix");
|
|
appendExtension(baseEnv, "acpx");
|
|
appendExtension(baseEnv, "codex");
|
|
|
|
const timingStore = await loadTimingStore(timingsFile, timingsEnabled);
|
|
const retriedMainLanes = applyLiveRetries(lanes, liveRetries);
|
|
const retriedTailLanes = applyLiveRetries(tailLanes, liveRetries);
|
|
const releaseLanes =
|
|
selectedLaneNames.length === 0 && profile === RELEASE_PATH_PROFILE
|
|
? releasePathChunkLanes(releaseChunk, { includeOpenWebUI })
|
|
: undefined;
|
|
const selectedLanes =
|
|
selectedLaneNames.length > 0
|
|
? selectNamedLanes(
|
|
dedupeLanes([
|
|
...allReleasePathLanes({ includeOpenWebUI }),
|
|
...retriedMainLanes,
|
|
...retriedTailLanes,
|
|
]),
|
|
selectedLaneNames,
|
|
"OPENCLAW_DOCKER_ALL_LANES",
|
|
)
|
|
: undefined;
|
|
const configuredLanes = selectedLanes
|
|
? selectedLanes
|
|
: releaseLanes
|
|
? releaseLanes
|
|
: liveMode === "only"
|
|
? applyLiveMode([...retriedMainLanes, ...retriedTailLanes], liveMode)
|
|
: applyLiveMode(retriedMainLanes, liveMode);
|
|
const configuredTailLanes =
|
|
selectedLanes || releaseLanes
|
|
? []
|
|
: liveMode === "only"
|
|
? []
|
|
: applyLiveMode(retriedTailLanes, liveMode);
|
|
const orderedLanes = orderLanes(configuredLanes, timingStore);
|
|
const orderedTailLanes = orderLanes(configuredTailLanes, timingStore);
|
|
|
|
console.log(`==> Docker test logs: ${logDir}`);
|
|
console.log(`==> Profile: ${profile}${releaseChunk ? ` chunk=${releaseChunk}` : ""}`);
|
|
console.log(`==> Parallelism: ${parallelism}`);
|
|
console.log(`==> Tail parallelism: ${tailParallelism}`);
|
|
console.log(`==> Lane timeout: ${laneTimeoutMs}ms`);
|
|
console.log(`==> Live mode: ${liveMode}`);
|
|
console.log(`==> Live retries: ${liveRetries}`);
|
|
console.log(`==> Lane start stagger: ${laneStartStaggerMs}ms`);
|
|
console.log(`==> Status interval: ${statusIntervalMs}ms`);
|
|
console.log(`==> Fail fast: ${failFast ? "yes" : "no"}`);
|
|
console.log(`==> Dry run: ${dryRun ? "yes" : "no"}`);
|
|
console.log(
|
|
`==> Docker preflight: ${preflightEnabled ? "yes" : "no"}${
|
|
preflightCleanup ? " cleanup=yes" : " cleanup=no"
|
|
}`,
|
|
);
|
|
console.log(`==> Build shared Docker images: ${buildEnabled ? "yes" : "no"}`);
|
|
if (profile === RELEASE_PATH_PROFILE) {
|
|
console.log(`==> Include Open WebUI: ${includeOpenWebUI ? "yes" : "no"}`);
|
|
}
|
|
if (selectedLaneNames.length > 0) {
|
|
console.log(`==> Selected lanes: ${selectedLaneNames.join(", ")}`);
|
|
}
|
|
console.log(`==> Docker lane timings: ${timingStore.enabled ? timingsFile : "disabled"}`);
|
|
console.log(`==> Live-test bundled plugin deps: ${baseEnv.OPENCLAW_DOCKER_BUILD_EXTENSIONS}`);
|
|
const schedulerOptions = parseSchedulerOptions(process.env, parallelism);
|
|
const tailSchedulerOptions = parseSchedulerOptions(process.env, tailParallelism);
|
|
console.log(
|
|
`==> Scheduler: weight=${schedulerOptions.weightLimit} ${resourceLimitsSummary(schedulerOptions.resourceLimits)}`,
|
|
);
|
|
console.log(
|
|
`==> Tail scheduler: weight=${tailSchedulerOptions.weightLimit} ${resourceLimitsSummary(tailSchedulerOptions.resourceLimits)}`,
|
|
);
|
|
printLaneManifest("Main", orderedLanes, timingStore);
|
|
printLaneManifest("Tail", orderedTailLanes, timingStore);
|
|
if (dryRun) {
|
|
console.log("==> Dry run complete");
|
|
return;
|
|
}
|
|
|
|
await runDockerPreflight(baseEnv, {
|
|
cleanup: preflightCleanup,
|
|
enabled: preflightEnabled,
|
|
runTimeoutMs: preflightRunTimeoutMs,
|
|
});
|
|
|
|
if (buildEnabled) {
|
|
const buildEntries = [];
|
|
const scheduledLanes = [...orderedLanes, ...orderedTailLanes];
|
|
if (scheduledLanes.some((poolLane) => poolLane.live)) {
|
|
buildEntries.push(["Build shared live-test image once", "pnpm test:docker:live-build"]);
|
|
}
|
|
if (scheduledLanes.some((poolLane) => !poolLane.live)) {
|
|
buildEntries.push([
|
|
`Build shared Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_IMAGE}`,
|
|
"pnpm test:docker:e2e-build",
|
|
]);
|
|
}
|
|
await runForegroundGroup(buildEntries, baseEnv);
|
|
} else {
|
|
console.log(`==> Shared Docker image builds: skipped`);
|
|
}
|
|
if (lanesNeedBundledPackage([...orderedLanes, ...orderedTailLanes])) {
|
|
await prepareBundledChannelPackage(baseEnv, logDir);
|
|
} else {
|
|
console.log("==> Bundled channel package: not needed for selected lanes");
|
|
}
|
|
|
|
const options = {
|
|
...schedulerOptions,
|
|
failFast,
|
|
poolLabel: "main",
|
|
startStaggerMs: laneStartStaggerMs,
|
|
statusIntervalMs,
|
|
timeoutMs: laneTimeoutMs,
|
|
};
|
|
const mainResult = await runLanePool(orderedLanes, baseEnv, logDir, parallelism, options);
|
|
const failures = [...mainResult.failures];
|
|
const allResults = [...mainResult.results];
|
|
await writeTimingStore(timingStore, mainResult.results);
|
|
if (failFast && failures.length > 0) {
|
|
await writeRunSummary(logDir, {
|
|
chunk: releaseChunk || undefined,
|
|
failures,
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
lanes: allResults,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "failed",
|
|
});
|
|
await printFailureSummary(failures, tailLines);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (orderedTailLanes.length > 0) {
|
|
console.log("==> Running provider-sensitive Docker tail lanes");
|
|
const tailResult = await runLanePool(orderedTailLanes, baseEnv, logDir, tailParallelism, {
|
|
...options,
|
|
...tailSchedulerOptions,
|
|
poolLabel: "tail",
|
|
});
|
|
failures.push(...tailResult.failures);
|
|
allResults.push(...tailResult.results);
|
|
await writeTimingStore(timingStore, tailResult.results);
|
|
} else {
|
|
console.log("==> Provider-sensitive Docker tail lanes: none");
|
|
}
|
|
if (failures.length > 0) {
|
|
await writeRunSummary(logDir, {
|
|
chunk: releaseChunk || undefined,
|
|
failures,
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
lanes: allResults,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "failed",
|
|
});
|
|
await printFailureSummary(failures, tailLines);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (profile === DEFAULT_PROFILE && selectedLaneNames.length === 0) {
|
|
await runForeground(
|
|
"Run cleanup smoke after parallel lanes",
|
|
"pnpm test:docker:cleanup",
|
|
baseEnv,
|
|
);
|
|
} else {
|
|
console.log("==> Cleanup smoke after parallel lanes: skipped for selected/release lanes");
|
|
}
|
|
await writeTimingStore(timingStore, allResults);
|
|
await writeRunSummary(logDir, {
|
|
chunk: releaseChunk || undefined,
|
|
failures,
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
lanes: allResults,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "passed",
|
|
});
|
|
console.log("==> Docker test suite passed");
|
|
}
|
|
|
|
await main().catch((error) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
});
|