mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-09 10:53:36 +02:00
perf: speed up local heavy checks
This commit is contained in:
@@ -132,7 +132,7 @@
|
||||
- Type-check/build: `pnpm build`
|
||||
- TypeScript checks: `pnpm tsgo`
|
||||
- Lint/format: `pnpm check`
|
||||
- Local agent/dev shells default to lower-memory `OPENCLAW_LOCAL_CHECK=1` behavior for `pnpm tsgo` and `pnpm lint`; set `OPENCLAW_LOCAL_CHECK=0` in CI/shared runs.
|
||||
- Local agent/dev shells default to host-aware `OPENCLAW_LOCAL_CHECK=1` behavior for `pnpm tsgo` and `pnpm lint`; set `OPENCLAW_LOCAL_CHECK_MODE=throttled` to force the lower-memory profile, `OPENCLAW_LOCAL_CHECK_MODE=full` to keep lock-only behavior, or `OPENCLAW_LOCAL_CHECK=0` in CI/shared runs.
|
||||
- Format check: `pnpm format` (oxfmt --check)
|
||||
- Format fix: `pnpm format:fix` (oxfmt --write)
|
||||
- Terminology:
|
||||
|
||||
@@ -3,12 +3,15 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const GIB = 1024 ** 3;
|
||||
const DEFAULT_LOCAL_GO_GC = "30";
|
||||
const DEFAULT_LOCAL_GO_MEMORY_LIMIT = "3GiB";
|
||||
const DEFAULT_LOCK_TIMEOUT_MS = 10 * 60 * 1000;
|
||||
const DEFAULT_LOCK_POLL_MS = 500;
|
||||
const DEFAULT_LOCK_PROGRESS_MS = 15 * 1000;
|
||||
const DEFAULT_STALE_LOCK_MS = 30 * 1000;
|
||||
const DEFAULT_FAST_LOCAL_CHECK_MIN_MEMORY_BYTES = 48 * GIB;
|
||||
const DEFAULT_FAST_LOCAL_CHECK_MIN_CPUS = 12;
|
||||
const SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4));
|
||||
|
||||
export function isLocalCheckEnabled(env) {
|
||||
@@ -20,7 +23,7 @@ export function hasFlag(args, name) {
|
||||
return args.some((arg) => arg === name || arg.startsWith(`${name}=`));
|
||||
}
|
||||
|
||||
export function applyLocalTsgoPolicy(args, env) {
|
||||
export function applyLocalTsgoPolicy(args, env, hostResources) {
|
||||
const nextEnv = { ...env };
|
||||
const nextArgs = [...args];
|
||||
|
||||
@@ -28,14 +31,16 @@ export function applyLocalTsgoPolicy(args, env) {
|
||||
return { env: nextEnv, args: nextArgs };
|
||||
}
|
||||
|
||||
insertBeforeSeparator(nextArgs, "--singleThreaded");
|
||||
insertBeforeSeparator(nextArgs, "--checkers", "1");
|
||||
if (shouldThrottleLocalHeavyChecks(nextEnv, hostResources)) {
|
||||
insertBeforeSeparator(nextArgs, "--singleThreaded");
|
||||
insertBeforeSeparator(nextArgs, "--checkers", "1");
|
||||
|
||||
if (!nextEnv.GOGC) {
|
||||
nextEnv.GOGC = DEFAULT_LOCAL_GO_GC;
|
||||
}
|
||||
if (!nextEnv.GOMEMLIMIT) {
|
||||
nextEnv.GOMEMLIMIT = DEFAULT_LOCAL_GO_MEMORY_LIMIT;
|
||||
if (!nextEnv.GOGC) {
|
||||
nextEnv.GOGC = DEFAULT_LOCAL_GO_GC;
|
||||
}
|
||||
if (!nextEnv.GOMEMLIMIT) {
|
||||
nextEnv.GOMEMLIMIT = DEFAULT_LOCAL_GO_MEMORY_LIMIT;
|
||||
}
|
||||
}
|
||||
if (nextEnv.OPENCLAW_TSGO_PPROF_DIR && !hasFlag(nextArgs, "--pprofDir")) {
|
||||
insertBeforeSeparator(nextArgs, "--pprofDir", nextEnv.OPENCLAW_TSGO_PPROF_DIR);
|
||||
@@ -44,20 +49,40 @@ export function applyLocalTsgoPolicy(args, env) {
|
||||
return { env: nextEnv, args: nextArgs };
|
||||
}
|
||||
|
||||
export function applyLocalOxlintPolicy(args, env) {
|
||||
export function applyLocalOxlintPolicy(args, env, hostResources) {
|
||||
const nextEnv = { ...env };
|
||||
const nextArgs = [...args];
|
||||
|
||||
insertBeforeSeparator(nextArgs, "--type-aware");
|
||||
insertBeforeSeparator(nextArgs, "--tsconfig", "tsconfig.oxlint.json");
|
||||
|
||||
if (isLocalCheckEnabled(nextEnv)) {
|
||||
if (shouldThrottleLocalHeavyChecks(nextEnv, hostResources)) {
|
||||
insertBeforeSeparator(nextArgs, "--threads=1");
|
||||
}
|
||||
|
||||
return { env: nextEnv, args: nextArgs };
|
||||
}
|
||||
|
||||
export function shouldThrottleLocalHeavyChecks(env, hostResources) {
|
||||
if (!isLocalCheckEnabled(env)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const mode = readLocalCheckMode(env);
|
||||
if (mode === "throttled") {
|
||||
return true;
|
||||
}
|
||||
if (mode === "full") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const resolvedHostResources = resolveHostResources(hostResources);
|
||||
return (
|
||||
resolvedHostResources.totalMemoryBytes < DEFAULT_FAST_LOCAL_CHECK_MIN_MEMORY_BYTES ||
|
||||
resolvedHostResources.logicalCpuCount < DEFAULT_FAST_LOCAL_CHECK_MIN_CPUS
|
||||
);
|
||||
}
|
||||
|
||||
export function acquireLocalHeavyCheckLockSync(params) {
|
||||
const env = params.env ?? process.env;
|
||||
|
||||
@@ -174,6 +199,29 @@ function insertBeforeSeparator(args, ...items) {
|
||||
args.splice(insertIndex, 0, ...items);
|
||||
}
|
||||
|
||||
function readLocalCheckMode(env) {
|
||||
const raw = env.OPENCLAW_LOCAL_CHECK_MODE?.trim().toLowerCase();
|
||||
if (raw === "throttled" || raw === "low-memory") {
|
||||
return "throttled";
|
||||
}
|
||||
if (raw === "full" || raw === "fast") {
|
||||
return "full";
|
||||
}
|
||||
return "auto";
|
||||
}
|
||||
|
||||
function resolveHostResources(hostResources) {
|
||||
if (hostResources) {
|
||||
return hostResources;
|
||||
}
|
||||
|
||||
return {
|
||||
totalMemoryBytes: os.totalmem(),
|
||||
logicalCpuCount:
|
||||
typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length,
|
||||
};
|
||||
}
|
||||
|
||||
function readPositiveInt(rawValue, fallback) {
|
||||
const parsed = Number.parseInt(rawValue ?? "", 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
|
||||
@@ -9,6 +9,15 @@ import {
|
||||
import { createScriptTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempDir } = createScriptTestHarness();
|
||||
const GIB = 1024 ** 3;
|
||||
const CONSTRAINED_HOST = {
|
||||
totalMemoryBytes: 16 * GIB,
|
||||
logicalCpuCount: 8,
|
||||
};
|
||||
const ROOMY_HOST = {
|
||||
totalMemoryBytes: 128 * GIB,
|
||||
logicalCpuCount: 16,
|
||||
};
|
||||
|
||||
function makeEnv(overrides: Record<string, string | undefined> = {}) {
|
||||
return {
|
||||
@@ -19,15 +28,15 @@ function makeEnv(overrides: Record<string, string | undefined> = {}) {
|
||||
}
|
||||
|
||||
describe("local-heavy-check-runtime", () => {
|
||||
it("tightens local tsgo runs to a single checker with a Go memory limit", () => {
|
||||
const { args, env } = applyLocalTsgoPolicy([], makeEnv());
|
||||
it("tightens local tsgo runs on constrained hosts", () => {
|
||||
const { args, env } = applyLocalTsgoPolicy([], makeEnv(), CONSTRAINED_HOST);
|
||||
|
||||
expect(args).toEqual(["--singleThreaded", "--checkers", "1"]);
|
||||
expect(env.GOGC).toBe("30");
|
||||
expect(env.GOMEMLIMIT).toBe("3GiB");
|
||||
});
|
||||
|
||||
it("keeps explicit tsgo flags and Go env overrides intact", () => {
|
||||
it("keeps explicit tsgo flags and Go env overrides intact when throttled", () => {
|
||||
const { args, env } = applyLocalTsgoPolicy(
|
||||
["--checkers", "4", "--singleThreaded", "--pprofDir", "/tmp/existing"],
|
||||
makeEnv({
|
||||
@@ -35,6 +44,7 @@ describe("local-heavy-check-runtime", () => {
|
||||
GOMEMLIMIT: "5GiB",
|
||||
OPENCLAW_TSGO_PPROF_DIR: "/tmp/profile",
|
||||
}),
|
||||
CONSTRAINED_HOST,
|
||||
);
|
||||
|
||||
expect(args).toEqual(["--checkers", "4", "--singleThreaded", "--pprofDir", "/tmp/existing"]);
|
||||
@@ -42,12 +52,40 @@ describe("local-heavy-check-runtime", () => {
|
||||
expect(env.GOMEMLIMIT).toBe("5GiB");
|
||||
});
|
||||
|
||||
it("serializes local oxlint runs onto one thread", () => {
|
||||
const { args } = applyLocalOxlintPolicy([], makeEnv());
|
||||
it("keeps local tsgo at full speed on roomy hosts in auto mode", () => {
|
||||
const { args, env } = applyLocalTsgoPolicy([], makeEnv(), ROOMY_HOST);
|
||||
|
||||
expect(args).toEqual([]);
|
||||
expect(env.GOGC).toBeUndefined();
|
||||
expect(env.GOMEMLIMIT).toBeUndefined();
|
||||
});
|
||||
|
||||
it("allows forcing the throttled tsgo policy on roomy hosts", () => {
|
||||
const { args, env } = applyLocalTsgoPolicy(
|
||||
[],
|
||||
makeEnv({
|
||||
OPENCLAW_LOCAL_CHECK_MODE: "throttled",
|
||||
}),
|
||||
ROOMY_HOST,
|
||||
);
|
||||
|
||||
expect(args).toEqual(["--singleThreaded", "--checkers", "1"]);
|
||||
expect(env.GOGC).toBe("30");
|
||||
expect(env.GOMEMLIMIT).toBe("3GiB");
|
||||
});
|
||||
|
||||
it("serializes local oxlint runs onto one thread on constrained hosts", () => {
|
||||
const { args } = applyLocalOxlintPolicy([], makeEnv(), CONSTRAINED_HOST);
|
||||
|
||||
expect(args).toEqual(["--type-aware", "--tsconfig", "tsconfig.oxlint.json", "--threads=1"]);
|
||||
});
|
||||
|
||||
it("keeps local oxlint parallel on roomy hosts in auto mode", () => {
|
||||
const { args } = applyLocalOxlintPolicy([], makeEnv(), ROOMY_HOST);
|
||||
|
||||
expect(args).toEqual(["--type-aware", "--tsconfig", "tsconfig.oxlint.json"]);
|
||||
});
|
||||
|
||||
it("reclaims stale local heavy-check locks from dead pids", () => {
|
||||
const cwd = createTempDir("openclaw-local-heavy-check-");
|
||||
const commonDir = path.join(cwd, ".git");
|
||||
|
||||
Reference in New Issue
Block a user