From 72c765e7368ee29cea3fb20d77cffed58db0211d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 22 Apr 2026 18:21:05 +0100 Subject: [PATCH] ci: parallelize additional boundary guards --- .github/workflows/ci.yml | 20 +-- docs/ci.md | 2 +- scripts/run-additional-boundary-checks.mjs | 161 ++++++++++++++++++ .../run-additional-boundary-checks.test.ts | 61 +++++++ 4 files changed, 225 insertions(+), 19 deletions(-) create mode 100644 scripts/run-additional-boundary-checks.mjs create mode 100644 test/scripts/run-additional-boundary-checks.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20285e1ac52..4444a47e856 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1707,6 +1707,7 @@ jobs: env: ADDITIONAL_CHECK_GROUP: ${{ matrix.group }} RUN_CONTROL_UI_I18N: ${{ needs.preflight.outputs.run_control_ui_i18n }} + OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY: 4 OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 6 shell: bash run: | @@ -1730,24 +1731,7 @@ jobs: case "$ADDITIONAL_CHECK_GROUP" in boundaries) - run_check "plugin-extension-boundary" pnpm run lint:plugins:no-extension-imports - run_check "lint:tmp:no-random-messaging" pnpm run lint:tmp:no-random-messaging - run_check "lint:tmp:channel-agnostic-boundaries" pnpm run lint:tmp:channel-agnostic-boundaries - run_check "lint:tmp:tsgo-core-boundary" pnpm run lint:tmp:tsgo-core-boundary - run_check "lint:tmp:no-raw-channel-fetch" pnpm run lint:tmp:no-raw-channel-fetch - run_check "lint:agent:ingress-owner" pnpm run lint:agent:ingress-owner - run_check "lint:plugins:no-register-http-handler" pnpm run lint:plugins:no-register-http-handler - run_check "lint:plugins:no-monolithic-plugin-sdk-entry-imports" pnpm run lint:plugins:no-monolithic-plugin-sdk-entry-imports - run_check "lint:plugins:no-extension-src-imports" pnpm run lint:plugins:no-extension-src-imports - run_check "lint:plugins:no-extension-test-core-imports" pnpm run lint:plugins:no-extension-test-core-imports - run_check "lint:plugins:plugin-sdk-subpaths-exported" pnpm run lint:plugins:plugin-sdk-subpaths-exported - run_check "deps:root-ownership:check" pnpm deps:root-ownership:check - run_check "web-search-provider-boundary" pnpm run lint:web-search-provider-boundaries - run_check "web-fetch-provider-boundary" pnpm run lint:web-fetch-provider-boundaries - run_check "extension-src-outside-plugin-sdk-boundary" pnpm run lint:extensions:no-src-outside-plugin-sdk - run_check "extension-plugin-sdk-internal-boundary" pnpm run lint:extensions:no-plugin-sdk-internal - run_check "extension-relative-outside-package-boundary" pnpm run lint:extensions:no-relative-outside-package - run_check "lint:ui:no-raw-window-open" pnpm lint:ui:no-raw-window-open + node scripts/run-additional-boundary-checks.mjs ;; extension-channels) run_check "lint:extensions:channels" pnpm run lint:extensions:channels diff --git a/docs/ci.md b/docs/ci.md index aeecfe5efe3..36456b99a35 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -52,7 +52,7 @@ Local changed-lane logic lives in `scripts/changed-lanes.mjs` and is executed by On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull requests, that lane is skipped and the matrix stays focused on the normal test/channel lanes. -The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into eight weighted shards each, auto-reply reply tests split by prefix group, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. `check-additional` keeps package-boundary compile/canary work together and separates it from runtime topology gateway/architecture work; the gateway watch regression uses the minimal `gatewayWatch` build profile instead of rebuilding the full CI artifact sidecar set. +The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into eight weighted shards each, auto-reply reply tests split by prefix group, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. `check-additional` keeps package-boundary compile/canary work together and separates it from runtime topology gateway/architecture work; the boundary guard shard runs its small independent guards concurrently inside one job, and the gateway watch regression uses the minimal `gatewayWatch` build profile instead of rebuilding the full CI artifact sidecar set. GitHub may mark superseded jobs as `cancelled` when a newer push lands on the same PR or `main` ref. Treat that as CI noise unless the newest run for the same ref is also failing. Aggregate shard checks use `!cancelled() && always()` so they still report normal shard failures but do not queue after the whole workflow has already been superseded. The CI concurrency key is versioned (`CI-v2-*`) so a GitHub-side zombie in an old queue group cannot indefinitely block newer main runs. diff --git a/scripts/run-additional-boundary-checks.mjs b/scripts/run-additional-boundary-checks.mjs new file mode 100644 index 00000000000..82e964e4090 --- /dev/null +++ b/scripts/run-additional-boundary-checks.mjs @@ -0,0 +1,161 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; + +export const BOUNDARY_CHECKS = [ + ["plugin-extension-boundary", "pnpm", ["run", "lint:plugins:no-extension-imports"]], + ["lint:tmp:no-random-messaging", "pnpm", ["run", "lint:tmp:no-random-messaging"]], + ["lint:tmp:channel-agnostic-boundaries", "pnpm", ["run", "lint:tmp:channel-agnostic-boundaries"]], + ["lint:tmp:tsgo-core-boundary", "pnpm", ["run", "lint:tmp:tsgo-core-boundary"]], + ["lint:tmp:no-raw-channel-fetch", "pnpm", ["run", "lint:tmp:no-raw-channel-fetch"]], + ["lint:agent:ingress-owner", "pnpm", ["run", "lint:agent:ingress-owner"]], + [ + "lint:plugins:no-register-http-handler", + "pnpm", + ["run", "lint:plugins:no-register-http-handler"], + ], + [ + "lint:plugins:no-monolithic-plugin-sdk-entry-imports", + "pnpm", + ["run", "lint:plugins:no-monolithic-plugin-sdk-entry-imports"], + ], + [ + "lint:plugins:no-extension-src-imports", + "pnpm", + ["run", "lint:plugins:no-extension-src-imports"], + ], + [ + "lint:plugins:no-extension-test-core-imports", + "pnpm", + ["run", "lint:plugins:no-extension-test-core-imports"], + ], + [ + "lint:plugins:plugin-sdk-subpaths-exported", + "pnpm", + ["run", "lint:plugins:plugin-sdk-subpaths-exported"], + ], + ["deps:root-ownership:check", "pnpm", ["deps:root-ownership:check"]], + ["web-search-provider-boundary", "pnpm", ["run", "lint:web-search-provider-boundaries"]], + ["web-fetch-provider-boundary", "pnpm", ["run", "lint:web-fetch-provider-boundaries"]], + [ + "extension-src-outside-plugin-sdk-boundary", + "pnpm", + ["run", "lint:extensions:no-src-outside-plugin-sdk"], + ], + [ + "extension-plugin-sdk-internal-boundary", + "pnpm", + ["run", "lint:extensions:no-plugin-sdk-internal"], + ], + [ + "extension-relative-outside-package-boundary", + "pnpm", + ["run", "lint:extensions:no-relative-outside-package"], + ], + ["lint:ui:no-raw-window-open", "pnpm", ["lint:ui:no-raw-window-open"]], +].map(([label, command, args]) => ({ label, command, args })); + +export function resolveConcurrency(value, fallback = 4) { + const parsed = Number.parseInt(String(value ?? ""), 10); + if (!Number.isFinite(parsed) || parsed < 1) { + return fallback; + } + return parsed; +} + +export function formatCommand({ command, args }) { + return [command, ...args].join(" "); +} + +function runSingleCheck(check, { cwd, env }) { + return new Promise((resolve) => { + const child = spawn(check.command, check.args, { + cwd, + env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + const chunks = []; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => chunks.push(chunk)); + child.stderr.on("data", (chunk) => chunks.push(chunk)); + child.on("error", (error) => { + chunks.push(`${error.stack ?? error.message}\n`); + resolve({ check, code: 1, signal: null, output: chunks.join("") }); + }); + child.on("close", (code, signal) => { + resolve({ check, code: code ?? 1, signal, output: chunks.join("") }); + }); + }); +} + +function writeGroupedResult(result, output) { + const success = result.code === 0; + output.write(`::group::${result.check.label}\n`); + output.write(`$ ${formatCommand(result.check)}\n`); + if (result.output) { + output.write(result.output.endsWith("\n") ? result.output : `${result.output}\n`); + } + if (success) { + output.write(`[ok] ${result.check.label}\n`); + } else { + const suffix = result.signal ? ` (signal ${result.signal})` : ` (exit ${result.code})`; + output.write( + `::error title=${result.check.label} failed::${result.check.label} failed${suffix}\n`, + ); + } + output.write("::endgroup::\n"); +} + +export async function runChecks( + checks = BOUNDARY_CHECKS, + { concurrency = 4, cwd = process.cwd(), env = process.env, output = process.stdout } = {}, +) { + const results = Array.from({ length: checks.length }); + let nextIndex = 0; + let active = 0; + + await new Promise((resolve) => { + const launch = () => { + if (nextIndex >= checks.length && active === 0) { + resolve(); + return; + } + + while (active < concurrency && nextIndex < checks.length) { + const index = nextIndex; + const check = checks[nextIndex++]; + active += 1; + void runSingleCheck(check, { cwd, env }) + .then((result) => { + results[index] = result; + }) + .finally(() => { + active -= 1; + launch(); + }); + } + }; + + launch(); + }); + + let failures = 0; + for (const result of results) { + writeGroupedResult(result, output); + if (result.code !== 0) { + failures += 1; + } + } + return failures; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const concurrency = resolveConcurrency( + process.env.OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY ?? + process.env.OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY, + ); + const failures = await runChecks(BOUNDARY_CHECKS, { concurrency }); + process.exitCode = failures === 0 ? 0 : 1; +} diff --git a/test/scripts/run-additional-boundary-checks.test.ts b/test/scripts/run-additional-boundary-checks.test.ts new file mode 100644 index 00000000000..41388e333e8 --- /dev/null +++ b/test/scripts/run-additional-boundary-checks.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { + formatCommand, + resolveConcurrency, + runChecks, +} from "../../scripts/run-additional-boundary-checks.mjs"; + +function createOutputBuffer() { + const chunks: string[] = []; + return { + output: { + write(chunk: string) { + chunks.push(chunk); + return true; + }, + }, + text: () => chunks.join(""), + }; +} + +describe("run-additional-boundary-checks", () => { + it("normalizes concurrency input", () => { + expect(resolveConcurrency("6")).toBe(6); + expect(resolveConcurrency("0")).toBe(4); + expect(resolveConcurrency("nope", 2)).toBe(2); + }); + + it("formats command display text", () => { + expect(formatCommand({ command: "pnpm", args: ["run", "lint:core"] })).toBe( + "pnpm run lint:core", + ); + }); + + it("buffers grouped output and reports aggregate failures", async () => { + const buffer = createOutputBuffer(); + const failures = await runChecks( + [ + { + label: "passes", + command: process.execPath, + args: ["-e", "console.log('ok-out')"], + }, + { + label: "fails", + command: process.execPath, + args: ["-e", "console.error('bad-out'); process.exit(7)"], + }, + ], + { concurrency: 2, output: buffer.output }, + ); + + const text = buffer.text(); + expect(failures).toBe(1); + expect(text).toContain("::group::passes"); + expect(text).toContain("ok-out"); + expect(text).toContain("[ok] passes"); + expect(text).toContain("::group::fails"); + expect(text).toContain("bad-out"); + expect(text).toContain("::error title=fails failed::fails failed (exit 7)"); + }); +});