From dbcee0bc76ccf7dd65cebb225ed5983399227c3e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 23 Apr 2026 08:40:09 +0100 Subject: [PATCH] ci: shard docker live model checks --- .../openclaw-live-and-e2e-checks-reusable.yml | 126 +++++++++++++++++- .github/workflows/openclaw-release-checks.yml | 1 + .../openclaw-scheduled-live-checks.yml | 1 + docs/help/testing.md | 8 ++ scripts/ci-hydrate-live-auth.sh | 3 +- 5 files changed, 133 insertions(+), 6 deletions(-) diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 0684979a749..cae0c0b6eac 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -141,6 +141,8 @@ on: required: false OPENCLAW_GEMINI_SETTINGS_JSON: required: false + FIREWORKS_API_KEY: + required: false permissions: contents: read @@ -472,6 +474,7 @@ jobs: OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} OPENCLAW_DOCKER_E2E_IMAGE: ${{ needs.prepare_docker_e2e_image.outputs.image }} OPENCLAW_SKIP_DOCKER_BUILD: "1" steps: @@ -623,6 +626,123 @@ jobs: provenance: false push: true + validate_live_models_docker: + name: Docker live models (${{ matrix.provider_label }}) + needs: validate_selected_ref + if: inputs.include_live_suites + runs-on: blacksmith-32vcpu-ubuntu-2404 + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - provider_label: Anthropic + providers: anthropic + - provider_label: Google + providers: google + - provider_label: MiniMax + providers: minimax + - provider_label: OpenAI + providers: openai + - provider_label: OpenCode + providers: opencode-go + - provider_label: OpenRouter + providers: openrouter + - provider_label: xAI + providers: xai + - provider_label: Z.ai + providers: zai + - provider_label: Fireworks + providers: fireworks + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }} + ANTHROPIC_API_KEY_OLD: ${{ secrets.ANTHROPIC_API_KEY_OLD }} + BYTEPLUS_API_KEY: ${{ secrets.BYTEPLUS_API_KEY }} + CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} + DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }} + MODELSTUDIO_API_KEY: ${{ secrets.MODELSTUDIO_API_KEY }} + MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + OPENCODE_ZEN_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} + Z_AI_API_KEY: ${{ secrets.Z_AI_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + OPENCLAW_CODEX_AUTH_JSON: ${{ secrets.OPENCLAW_CODEX_AUTH_JSON }} + OPENCLAW_CODEX_CONFIG_TOML: ${{ secrets.OPENCLAW_CODEX_CONFIG_TOML }} + OPENCLAW_CLAUDE_JSON: ${{ secrets.OPENCLAW_CLAUDE_JSON }} + OPENCLAW_CLAUDE_CREDENTIALS_JSON: ${{ secrets.OPENCLAW_CLAUDE_CREDENTIALS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} + OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + OPENCLAW_LIVE_PROVIDERS: ${{ matrix.providers }} + OPENCLAW_VITEST_MAX_WORKERS: "2" + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Hydrate live auth/profile inputs + run: bash scripts/ci-hydrate-live-auth.sh + + - name: Validate provider credential + shell: bash + run: | + set -euo pipefail + + require_any() { + local label="$1" + shift + local key + for key in "$@"; do + if [[ -n "${!key:-}" ]]; then + return 0 + fi + done + echo "Missing credential for ${label}: expected one of $*" >&2 + exit 1 + } + + case "${{ matrix.providers }}" in + anthropic) require_any Anthropic ANTHROPIC_API_KEY ANTHROPIC_API_KEY_OLD ANTHROPIC_API_TOKEN ;; + google) require_any Google GEMINI_API_KEY GOOGLE_API_KEY ;; + minimax) require_any MiniMax MINIMAX_API_KEY ;; + openai) require_any OpenAI OPENAI_API_KEY ;; + opencode-go) require_any OpenCode OPENCODE_API_KEY OPENCODE_ZEN_API_KEY ;; + openrouter) require_any OpenRouter OPENROUTER_API_KEY ;; + xai) require_any xAI XAI_API_KEY ;; + zai) require_any Z.ai ZAI_API_KEY Z_AI_API_KEY ;; + fireworks) require_any Fireworks FIREWORKS_API_KEY ;; + *) + echo "Unhandled live model provider shard: ${{ matrix.providers }}" >&2 + exit 1 + ;; + esac + + - name: Run Docker live model sweep + run: pnpm test:docker:live-models + validate_live_provider_suites: needs: validate_selected_ref if: inputs.include_live_suites @@ -637,11 +757,6 @@ jobs: command: pnpm test:live timeout_minutes: 180 profile_env_only: false - - suite_id: live-models-docker - label: Docker live models - command: pnpm test:docker:live-models - timeout_minutes: 120 - profile_env_only: false - suite_id: live-gateway-docker label: Docker live gateway command: pnpm test:docker:live-gateway @@ -706,6 +821,7 @@ jobs: OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} OPENCLAW_LIVE_VIDEO_GENERATION_SKIP_PROVIDERS: "" OPENCLAW_LIVE_VYDRA_VIDEO: "1" OPENCLAW_VITEST_MAX_WORKERS: "2" diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 3cc744c4854..bb39d1f46d1 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -199,6 +199,7 @@ jobs: OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} qa_lab_parity_release_checks: name: Run QA Lab parity gate diff --git a/.github/workflows/openclaw-scheduled-live-checks.yml b/.github/workflows/openclaw-scheduled-live-checks.yml index d58980a51f9..14b2c84635b 100644 --- a/.github/workflows/openclaw-scheduled-live-checks.yml +++ b/.github/workflows/openclaw-scheduled-live-checks.yml @@ -74,3 +74,4 @@ jobs: OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} diff --git a/docs/help/testing.md b/docs/help/testing.md index c5ca5183dfc..ffced1e0f86 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -39,6 +39,14 @@ When debugging real providers/models (requires real creds): - Live suite (models + gateway tool/image probes): `pnpm test:live` - Target one live file quietly: `pnpm test:live -- src/agents/models.profiles.live.test.ts` +- Docker live model sweep: `pnpm test:docker:live-models` + - CI coverage: daily `OpenClaw Scheduled Live And E2E Checks` and manual + `OpenClaw Release Checks` both call the reusable live/E2E workflow with + `include_live_suites: true`, which includes separate Docker live model + matrix jobs sharded by provider. + - Add new high-signal provider secrets to `scripts/ci-hydrate-live-auth.sh` + plus `.github/workflows/openclaw-live-and-e2e-checks-reusable.yml` and its + scheduled/release callers. - Moonshot/Kimi cost smoke: with `MOONSHOT_API_KEY` set, run `openclaw models list --provider moonshot --json`, then run an isolated `openclaw agent --local --session-id live-kimi-cost --message 'Reply exactly: KIMI_LIVE_OK' --thinking off --json` diff --git a/scripts/ci-hydrate-live-auth.sh b/scripts/ci-hydrate-live-auth.sh index a1b2123a4f4..e2462a5e07c 100644 --- a/scripts/ci-hydrate-live-auth.sh +++ b/scripts/ci-hydrate-live-auth.sh @@ -64,7 +64,8 @@ for env_key in \ Z_AI_API_KEY \ BYTEPLUS_ACCESS_KEY_ID \ BYTEPLUS_SECRET_ACCESS_KEY \ - CLAUDE_CODE_OAUTH_TOKEN + CLAUDE_CODE_OAUTH_TOKEN \ + FIREWORKS_API_KEY do append_profile_env "$env_key" done