mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-28 20:46:57 +02:00
ci: parallelize additional boundary guards
This commit is contained in:
20
.github/workflows/ci.yml
vendored
20
.github/workflows/ci.yml
vendored
@@ -1707,6 +1707,7 @@ jobs:
|
||||
env:
|
||||
ADDITIONAL_CHECK_GROUP: ${{ matrix.group }}
|
||||
RUN_CONTROL_UI_I18N: ${{ needs.preflight.outputs.run_control_ui_i18n }}
|
||||
OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY: 4
|
||||
OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 6
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -1730,24 +1731,7 @@ jobs:
|
||||
|
||||
case "$ADDITIONAL_CHECK_GROUP" in
|
||||
boundaries)
|
||||
run_check "plugin-extension-boundary" pnpm run lint:plugins:no-extension-imports
|
||||
run_check "lint:tmp:no-random-messaging" pnpm run lint:tmp:no-random-messaging
|
||||
run_check "lint:tmp:channel-agnostic-boundaries" pnpm run lint:tmp:channel-agnostic-boundaries
|
||||
run_check "lint:tmp:tsgo-core-boundary" pnpm run lint:tmp:tsgo-core-boundary
|
||||
run_check "lint:tmp:no-raw-channel-fetch" pnpm run lint:tmp:no-raw-channel-fetch
|
||||
run_check "lint:agent:ingress-owner" pnpm run lint:agent:ingress-owner
|
||||
run_check "lint:plugins:no-register-http-handler" pnpm run lint:plugins:no-register-http-handler
|
||||
run_check "lint:plugins:no-monolithic-plugin-sdk-entry-imports" pnpm run lint:plugins:no-monolithic-plugin-sdk-entry-imports
|
||||
run_check "lint:plugins:no-extension-src-imports" pnpm run lint:plugins:no-extension-src-imports
|
||||
run_check "lint:plugins:no-extension-test-core-imports" pnpm run lint:plugins:no-extension-test-core-imports
|
||||
run_check "lint:plugins:plugin-sdk-subpaths-exported" pnpm run lint:plugins:plugin-sdk-subpaths-exported
|
||||
run_check "deps:root-ownership:check" pnpm deps:root-ownership:check
|
||||
run_check "web-search-provider-boundary" pnpm run lint:web-search-provider-boundaries
|
||||
run_check "web-fetch-provider-boundary" pnpm run lint:web-fetch-provider-boundaries
|
||||
run_check "extension-src-outside-plugin-sdk-boundary" pnpm run lint:extensions:no-src-outside-plugin-sdk
|
||||
run_check "extension-plugin-sdk-internal-boundary" pnpm run lint:extensions:no-plugin-sdk-internal
|
||||
run_check "extension-relative-outside-package-boundary" pnpm run lint:extensions:no-relative-outside-package
|
||||
run_check "lint:ui:no-raw-window-open" pnpm lint:ui:no-raw-window-open
|
||||
node scripts/run-additional-boundary-checks.mjs
|
||||
;;
|
||||
extension-channels)
|
||||
run_check "lint:extensions:channels" pnpm run lint:extensions:channels
|
||||
|
||||
@@ -52,7 +52,7 @@ Local changed-lane logic lives in `scripts/changed-lanes.mjs` and is executed by
|
||||
|
||||
On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull requests, that lane is skipped and the matrix stays focused on the normal test/channel lanes.
|
||||
|
||||
The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into eight weighted shards each, auto-reply reply tests split by prefix group, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. `check-additional` keeps package-boundary compile/canary work together and separates it from runtime topology gateway/architecture work; the gateway watch regression uses the minimal `gatewayWatch` build profile instead of rebuilding the full CI artifact sidecar set.
|
||||
The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into eight weighted shards each, auto-reply reply tests split by prefix group, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. `check-additional` keeps package-boundary compile/canary work together and separates it from runtime topology gateway/architecture work; the boundary guard shard runs its small independent guards concurrently inside one job, and the gateway watch regression uses the minimal `gatewayWatch` build profile instead of rebuilding the full CI artifact sidecar set.
|
||||
|
||||
GitHub may mark superseded jobs as `cancelled` when a newer push lands on the same PR or `main` ref. Treat that as CI noise unless the newest run for the same ref is also failing. Aggregate shard checks use `!cancelled() && always()` so they still report normal shard failures but do not queue after the whole workflow has already been superseded.
|
||||
The CI concurrency key is versioned (`CI-v2-*`) so a GitHub-side zombie in an old queue group cannot indefinitely block newer main runs.
|
||||
|
||||
161
scripts/run-additional-boundary-checks.mjs
Normal file
161
scripts/run-additional-boundary-checks.mjs
Normal file
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
export const BOUNDARY_CHECKS = [
|
||||
["plugin-extension-boundary", "pnpm", ["run", "lint:plugins:no-extension-imports"]],
|
||||
["lint:tmp:no-random-messaging", "pnpm", ["run", "lint:tmp:no-random-messaging"]],
|
||||
["lint:tmp:channel-agnostic-boundaries", "pnpm", ["run", "lint:tmp:channel-agnostic-boundaries"]],
|
||||
["lint:tmp:tsgo-core-boundary", "pnpm", ["run", "lint:tmp:tsgo-core-boundary"]],
|
||||
["lint:tmp:no-raw-channel-fetch", "pnpm", ["run", "lint:tmp:no-raw-channel-fetch"]],
|
||||
["lint:agent:ingress-owner", "pnpm", ["run", "lint:agent:ingress-owner"]],
|
||||
[
|
||||
"lint:plugins:no-register-http-handler",
|
||||
"pnpm",
|
||||
["run", "lint:plugins:no-register-http-handler"],
|
||||
],
|
||||
[
|
||||
"lint:plugins:no-monolithic-plugin-sdk-entry-imports",
|
||||
"pnpm",
|
||||
["run", "lint:plugins:no-monolithic-plugin-sdk-entry-imports"],
|
||||
],
|
||||
[
|
||||
"lint:plugins:no-extension-src-imports",
|
||||
"pnpm",
|
||||
["run", "lint:plugins:no-extension-src-imports"],
|
||||
],
|
||||
[
|
||||
"lint:plugins:no-extension-test-core-imports",
|
||||
"pnpm",
|
||||
["run", "lint:plugins:no-extension-test-core-imports"],
|
||||
],
|
||||
[
|
||||
"lint:plugins:plugin-sdk-subpaths-exported",
|
||||
"pnpm",
|
||||
["run", "lint:plugins:plugin-sdk-subpaths-exported"],
|
||||
],
|
||||
["deps:root-ownership:check", "pnpm", ["deps:root-ownership:check"]],
|
||||
["web-search-provider-boundary", "pnpm", ["run", "lint:web-search-provider-boundaries"]],
|
||||
["web-fetch-provider-boundary", "pnpm", ["run", "lint:web-fetch-provider-boundaries"]],
|
||||
[
|
||||
"extension-src-outside-plugin-sdk-boundary",
|
||||
"pnpm",
|
||||
["run", "lint:extensions:no-src-outside-plugin-sdk"],
|
||||
],
|
||||
[
|
||||
"extension-plugin-sdk-internal-boundary",
|
||||
"pnpm",
|
||||
["run", "lint:extensions:no-plugin-sdk-internal"],
|
||||
],
|
||||
[
|
||||
"extension-relative-outside-package-boundary",
|
||||
"pnpm",
|
||||
["run", "lint:extensions:no-relative-outside-package"],
|
||||
],
|
||||
["lint:ui:no-raw-window-open", "pnpm", ["lint:ui:no-raw-window-open"]],
|
||||
].map(([label, command, args]) => ({ label, command, args }));
|
||||
|
||||
export function resolveConcurrency(value, fallback = 4) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
return fallback;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function formatCommand({ command, args }) {
|
||||
return [command, ...args].join(" ");
|
||||
}
|
||||
|
||||
function runSingleCheck(check, { cwd, env }) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(check.command, check.args, {
|
||||
cwd,
|
||||
env,
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const chunks = [];
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => chunks.push(chunk));
|
||||
child.stderr.on("data", (chunk) => chunks.push(chunk));
|
||||
child.on("error", (error) => {
|
||||
chunks.push(`${error.stack ?? error.message}\n`);
|
||||
resolve({ check, code: 1, signal: null, output: chunks.join("") });
|
||||
});
|
||||
child.on("close", (code, signal) => {
|
||||
resolve({ check, code: code ?? 1, signal, output: chunks.join("") });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function writeGroupedResult(result, output) {
|
||||
const success = result.code === 0;
|
||||
output.write(`::group::${result.check.label}\n`);
|
||||
output.write(`$ ${formatCommand(result.check)}\n`);
|
||||
if (result.output) {
|
||||
output.write(result.output.endsWith("\n") ? result.output : `${result.output}\n`);
|
||||
}
|
||||
if (success) {
|
||||
output.write(`[ok] ${result.check.label}\n`);
|
||||
} else {
|
||||
const suffix = result.signal ? ` (signal ${result.signal})` : ` (exit ${result.code})`;
|
||||
output.write(
|
||||
`::error title=${result.check.label} failed::${result.check.label} failed${suffix}\n`,
|
||||
);
|
||||
}
|
||||
output.write("::endgroup::\n");
|
||||
}
|
||||
|
||||
export async function runChecks(
|
||||
checks = BOUNDARY_CHECKS,
|
||||
{ concurrency = 4, cwd = process.cwd(), env = process.env, output = process.stdout } = {},
|
||||
) {
|
||||
const results = Array.from({ length: checks.length });
|
||||
let nextIndex = 0;
|
||||
let active = 0;
|
||||
|
||||
await new Promise((resolve) => {
|
||||
const launch = () => {
|
||||
if (nextIndex >= checks.length && active === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
while (active < concurrency && nextIndex < checks.length) {
|
||||
const index = nextIndex;
|
||||
const check = checks[nextIndex++];
|
||||
active += 1;
|
||||
void runSingleCheck(check, { cwd, env })
|
||||
.then((result) => {
|
||||
results[index] = result;
|
||||
})
|
||||
.finally(() => {
|
||||
active -= 1;
|
||||
launch();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
launch();
|
||||
});
|
||||
|
||||
let failures = 0;
|
||||
for (const result of results) {
|
||||
writeGroupedResult(result, output);
|
||||
if (result.code !== 0) {
|
||||
failures += 1;
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const concurrency = resolveConcurrency(
|
||||
process.env.OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY ??
|
||||
process.env.OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY,
|
||||
);
|
||||
const failures = await runChecks(BOUNDARY_CHECKS, { concurrency });
|
||||
process.exitCode = failures === 0 ? 0 : 1;
|
||||
}
|
||||
61
test/scripts/run-additional-boundary-checks.test.ts
Normal file
61
test/scripts/run-additional-boundary-checks.test.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formatCommand,
|
||||
resolveConcurrency,
|
||||
runChecks,
|
||||
} from "../../scripts/run-additional-boundary-checks.mjs";
|
||||
|
||||
function createOutputBuffer() {
|
||||
const chunks: string[] = [];
|
||||
return {
|
||||
output: {
|
||||
write(chunk: string) {
|
||||
chunks.push(chunk);
|
||||
return true;
|
||||
},
|
||||
},
|
||||
text: () => chunks.join(""),
|
||||
};
|
||||
}
|
||||
|
||||
describe("run-additional-boundary-checks", () => {
|
||||
it("normalizes concurrency input", () => {
|
||||
expect(resolveConcurrency("6")).toBe(6);
|
||||
expect(resolveConcurrency("0")).toBe(4);
|
||||
expect(resolveConcurrency("nope", 2)).toBe(2);
|
||||
});
|
||||
|
||||
it("formats command display text", () => {
|
||||
expect(formatCommand({ command: "pnpm", args: ["run", "lint:core"] })).toBe(
|
||||
"pnpm run lint:core",
|
||||
);
|
||||
});
|
||||
|
||||
it("buffers grouped output and reports aggregate failures", async () => {
|
||||
const buffer = createOutputBuffer();
|
||||
const failures = await runChecks(
|
||||
[
|
||||
{
|
||||
label: "passes",
|
||||
command: process.execPath,
|
||||
args: ["-e", "console.log('ok-out')"],
|
||||
},
|
||||
{
|
||||
label: "fails",
|
||||
command: process.execPath,
|
||||
args: ["-e", "console.error('bad-out'); process.exit(7)"],
|
||||
},
|
||||
],
|
||||
{ concurrency: 2, output: buffer.output },
|
||||
);
|
||||
|
||||
const text = buffer.text();
|
||||
expect(failures).toBe(1);
|
||||
expect(text).toContain("::group::passes");
|
||||
expect(text).toContain("ok-out");
|
||||
expect(text).toContain("[ok] passes");
|
||||
expect(text).toContain("::group::fails");
|
||||
expect(text).toContain("bad-out");
|
||||
expect(text).toContain("::error title=fails failed::fails failed (exit 7)");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user