Models: harden SecretRef-managed apiKey persistence
This commit is contained in:
joshavant
parent
bdd0f74188
commit
4dd006a6a3
44
src/agents/model-auth-markers.ts
Normal file
44
src/agents/model-auth-markers.ts
Normal file
@ -0,0 +1,44 @@
|
||||
import type { SecretRefSource } from "../config/types.secrets.js";
|
||||
|
||||
export const MINIMAX_OAUTH_MARKER = "minimax-oauth";
|
||||
export const QWEN_OAUTH_MARKER = "qwen-oauth";
|
||||
export const OLLAMA_LOCAL_AUTH_MARKER = "ollama-local";
|
||||
export const NON_ENV_SECRETREF_MARKER = "secretref-managed";
|
||||
|
||||
const AWS_SDK_ENV_MARKERS = new Set([
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_PROFILE",
|
||||
]);
|
||||
|
||||
export function isEnvVarNameMarker(value: string): boolean {
|
||||
return /^[A-Z][A-Z0-9_]*$/.test(value.trim());
|
||||
}
|
||||
|
||||
export function isAwsSdkAuthMarker(value: string): boolean {
|
||||
return AWS_SDK_ENV_MARKERS.has(value.trim());
|
||||
}
|
||||
|
||||
export function resolveNonEnvSecretRefApiKeyMarker(_source: SecretRefSource): string {
|
||||
return NON_ENV_SECRETREF_MARKER;
|
||||
}
|
||||
|
||||
export function isNonSecretApiKeyMarker(
|
||||
value: string,
|
||||
opts?: { includeEnvVarName?: boolean },
|
||||
): boolean {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
const isKnownMarker =
|
||||
trimmed === MINIMAX_OAUTH_MARKER ||
|
||||
trimmed === QWEN_OAUTH_MARKER ||
|
||||
trimmed === OLLAMA_LOCAL_AUTH_MARKER ||
|
||||
trimmed === NON_ENV_SECRETREF_MARKER ||
|
||||
isAwsSdkAuthMarker(trimmed);
|
||||
if (isKnownMarker) {
|
||||
return true;
|
||||
}
|
||||
return opts?.includeEnvVarName === false ? false : isEnvVarNameMarker(trimmed);
|
||||
}
|
||||
@ -16,6 +16,7 @@ import {
|
||||
resolveAuthProfileOrder,
|
||||
resolveAuthStorePathForDisplay,
|
||||
} from "./auth-profiles.js";
|
||||
import { OLLAMA_LOCAL_AUTH_MARKER } from "./model-auth-markers.js";
|
||||
import { normalizeProviderId } from "./model-selection.js";
|
||||
|
||||
export { ensureAuthProfileStore, resolveAuthProfileOrder } from "./auth-profiles.js";
|
||||
@ -90,7 +91,7 @@ function resolveSyntheticLocalProviderAuth(params: {
|
||||
}
|
||||
|
||||
return {
|
||||
apiKey: "ollama-local", // pragma: allowlist secret
|
||||
apiKey: OLLAMA_LOCAL_AUTH_MARKER,
|
||||
source: "models.providers.ollama (synthetic local key)",
|
||||
mode: "api-key",
|
||||
};
|
||||
|
||||
43
src/agents/models-config.file-mode.test.ts
Normal file
43
src/agents/models-config.file-mode.test.ts
Normal file
@ -0,0 +1,43 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveOpenClawAgentDir } from "./agent-paths.js";
|
||||
import {
|
||||
CUSTOM_PROXY_MODELS_CONFIG,
|
||||
installModelsConfigTestHooks,
|
||||
withModelsTempHome as withTempHome,
|
||||
} from "./models-config.e2e-harness.js";
|
||||
import { ensureOpenClawModelsJson } from "./models-config.js";
|
||||
|
||||
installModelsConfigTestHooks();
|
||||
|
||||
describe("models-config file mode", () => {
|
||||
it("writes models.json with mode 0600", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
await withTempHome(async () => {
|
||||
await ensureOpenClawModelsJson(CUSTOM_PROXY_MODELS_CONFIG);
|
||||
const modelsPath = path.join(resolveOpenClawAgentDir(), "models.json");
|
||||
const stat = await fs.stat(modelsPath);
|
||||
expect(stat.mode & 0o777).toBe(0o600);
|
||||
});
|
||||
});
|
||||
|
||||
it("repairs models.json mode to 0600 on no-content-change paths", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
await withTempHome(async () => {
|
||||
await ensureOpenClawModelsJson(CUSTOM_PROXY_MODELS_CONFIG);
|
||||
const modelsPath = path.join(resolveOpenClawAgentDir(), "models.json");
|
||||
await fs.chmod(modelsPath, 0o644);
|
||||
|
||||
const result = await ensureOpenClawModelsJson(CUSTOM_PROXY_MODELS_CONFIG);
|
||||
expect(result.wrote).toBe(false);
|
||||
|
||||
const stat = await fs.stat(modelsPath);
|
||||
expect(stat.mode & 0o777).toBe(0o600);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -220,6 +220,85 @@ describe("models-config", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("replaces stale merged apiKey when provider is SecretRef-managed in current config", async () => {
|
||||
await withTempHome(async () => {
|
||||
await writeAgentModelsJson({
|
||||
providers: {
|
||||
custom: {
|
||||
baseUrl: "https://agent.example/v1",
|
||||
apiKey: "STALE_AGENT_KEY",
|
||||
api: "openai-responses",
|
||||
models: [{ id: "agent-model", name: "Agent model", input: ["text"] }],
|
||||
},
|
||||
},
|
||||
});
|
||||
await ensureOpenClawModelsJson({
|
||||
models: {
|
||||
mode: "merge",
|
||||
providers: {
|
||||
custom: {
|
||||
...createMergeConfigProvider(),
|
||||
apiKey: { source: "env", provider: "default", id: "CUSTOM_PROVIDER_API_KEY" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const parsed = await readGeneratedModelsJson<{
|
||||
providers: Record<string, { apiKey?: string; baseUrl?: string }>;
|
||||
}>();
|
||||
expect(parsed.providers.custom?.apiKey).toBe("CUSTOM_PROVIDER_API_KEY");
|
||||
expect(parsed.providers.custom?.baseUrl).toBe("https://agent.example/v1");
|
||||
});
|
||||
});
|
||||
|
||||
it("replaces stale merged apiKey when provider is SecretRef-managed via auth-profiles", async () => {
|
||||
await withTempHome(async () => {
|
||||
const agentDir = resolveOpenClawAgentDir();
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(agentDir, "auth-profiles.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"minimax:default": {
|
||||
type: "api_key",
|
||||
provider: "minimax",
|
||||
keyRef: { source: "env", provider: "default", id: "MINIMAX_API_KEY" },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeAgentModelsJson({
|
||||
providers: {
|
||||
minimax: {
|
||||
baseUrl: "https://api.minimax.io/anthropic",
|
||||
apiKey: "STALE_AGENT_KEY",
|
||||
api: "anthropic-messages",
|
||||
models: [{ id: "MiniMax-M2.5", name: "MiniMax M2.5", input: ["text"] }],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await ensureOpenClawModelsJson({
|
||||
models: {
|
||||
mode: "merge",
|
||||
providers: {},
|
||||
},
|
||||
});
|
||||
|
||||
const parsed = await readGeneratedModelsJson<{
|
||||
providers: Record<string, { apiKey?: string }>;
|
||||
}>();
|
||||
expect(parsed.providers.minimax?.apiKey).toBe("MINIMAX_API_KEY");
|
||||
});
|
||||
});
|
||||
|
||||
it("uses config apiKey/baseUrl when existing agent values are empty", async () => {
|
||||
await withTempHome(async () => {
|
||||
const parsed = await runCustomProviderMergeTest({
|
||||
|
||||
121
src/agents/models-config.providers.auth-provenance.test.ts
Normal file
121
src/agents/models-config.providers.auth-provenance.test.ts
Normal file
@ -0,0 +1,121 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import {
|
||||
MINIMAX_OAUTH_MARKER,
|
||||
NON_ENV_SECRETREF_MARKER,
|
||||
QWEN_OAUTH_MARKER,
|
||||
} from "./model-auth-markers.js";
|
||||
import { resolveImplicitProviders } from "./models-config.providers.js";
|
||||
|
||||
describe("models-config provider auth provenance", () => {
|
||||
it("persists env keyRef and tokenRef auth profiles as env var markers", async () => {
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
|
||||
const envSnapshot = captureEnv(["VOLCANO_ENGINE_API_KEY", "TOGETHER_API_KEY"]);
|
||||
delete process.env.VOLCANO_ENGINE_API_KEY;
|
||||
delete process.env.TOGETHER_API_KEY;
|
||||
await writeFile(
|
||||
join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"volcengine:default": {
|
||||
type: "api_key",
|
||||
provider: "volcengine",
|
||||
keyRef: { source: "env", provider: "default", id: "VOLCANO_ENGINE_API_KEY" },
|
||||
},
|
||||
"together:default": {
|
||||
type: "token",
|
||||
provider: "together",
|
||||
tokenRef: { source: "env", provider: "default", id: "TOGETHER_API_KEY" },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
try {
|
||||
const providers = await resolveImplicitProviders({ agentDir });
|
||||
expect(providers?.volcengine?.apiKey).toBe("VOLCANO_ENGINE_API_KEY");
|
||||
expect(providers?.["volcengine-plan"]?.apiKey).toBe("VOLCANO_ENGINE_API_KEY");
|
||||
expect(providers?.together?.apiKey).toBe("TOGETHER_API_KEY");
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("uses non-env marker for ref-managed profiles even when runtime plaintext is present", async () => {
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
|
||||
await writeFile(
|
||||
join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"byteplus:default": {
|
||||
type: "api_key",
|
||||
provider: "byteplus",
|
||||
key: "sk-runtime-resolved-byteplus",
|
||||
keyRef: { source: "file", provider: "vault", id: "/byteplus/apiKey" },
|
||||
},
|
||||
"together:default": {
|
||||
type: "token",
|
||||
provider: "together",
|
||||
token: "tok-runtime-resolved-together",
|
||||
tokenRef: { source: "exec", provider: "vault", id: "providers/together/token" },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const providers = await resolveImplicitProviders({ agentDir });
|
||||
expect(providers?.byteplus?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
|
||||
expect(providers?.["byteplus-plan"]?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
|
||||
expect(providers?.together?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
|
||||
});
|
||||
|
||||
it("keeps oauth compatibility markers for minimax-portal and qwen-portal", async () => {
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
|
||||
await writeFile(
|
||||
join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"minimax-portal:default": {
|
||||
type: "oauth",
|
||||
provider: "minimax-portal",
|
||||
access: "access-token",
|
||||
refresh: "refresh-token",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
"qwen-portal:default": {
|
||||
type: "oauth",
|
||||
provider: "qwen-portal",
|
||||
access: "access-token",
|
||||
refresh: "refresh-token",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const providers = await resolveImplicitProviders({ agentDir });
|
||||
expect(providers?.["minimax-portal"]?.apiKey).toBe(MINIMAX_OAUTH_MARKER);
|
||||
expect(providers?.["qwen-portal"]?.apiKey).toBe(QWEN_OAUTH_MARKER);
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,76 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
|
||||
import { resolveImplicitProviders } from "./models-config.providers.js";
|
||||
|
||||
describe("cloudflare-ai-gateway profile provenance", () => {
|
||||
it("prefers env keyRef marker over runtime plaintext for persistence", async () => {
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
|
||||
const envSnapshot = captureEnv(["CLOUDFLARE_AI_GATEWAY_API_KEY"]);
|
||||
delete process.env.CLOUDFLARE_AI_GATEWAY_API_KEY;
|
||||
|
||||
await writeFile(
|
||||
join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"cloudflare-ai-gateway:default": {
|
||||
type: "api_key",
|
||||
provider: "cloudflare-ai-gateway",
|
||||
key: "sk-runtime-cloudflare",
|
||||
keyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
|
||||
metadata: {
|
||||
accountId: "acct_123",
|
||||
gatewayId: "gateway_456",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
try {
|
||||
const providers = await resolveImplicitProviders({ agentDir });
|
||||
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe("CLOUDFLARE_AI_GATEWAY_API_KEY");
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("uses non-env marker for non-env keyRef cloudflare profiles", async () => {
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
|
||||
await writeFile(
|
||||
join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"cloudflare-ai-gateway:default": {
|
||||
type: "api_key",
|
||||
provider: "cloudflare-ai-gateway",
|
||||
key: "sk-runtime-cloudflare",
|
||||
keyRef: { source: "file", provider: "vault", id: "/cloudflare/apiKey" },
|
||||
metadata: {
|
||||
accountId: "acct_123",
|
||||
gatewayId: "gateway_456",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const providers = await resolveImplicitProviders({ agentDir });
|
||||
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
|
||||
});
|
||||
});
|
||||
106
src/agents/models-config.providers.discovery-auth.test.ts
Normal file
106
src/agents/models-config.providers.discovery-auth.test.ts
Normal file
@ -0,0 +1,106 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
|
||||
import { resolveImplicitProviders } from "./models-config.providers.js";
|
||||
|
||||
describe("provider discovery auth marker guardrails", () => {
|
||||
let originalVitest: string | undefined;
|
||||
let originalNodeEnv: string | undefined;
|
||||
let originalFetch: typeof globalThis.fetch | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
if (originalVitest !== undefined) {
|
||||
process.env.VITEST = originalVitest;
|
||||
} else {
|
||||
delete process.env.VITEST;
|
||||
}
|
||||
if (originalNodeEnv !== undefined) {
|
||||
process.env.NODE_ENV = originalNodeEnv;
|
||||
} else {
|
||||
delete process.env.NODE_ENV;
|
||||
}
|
||||
if (originalFetch) {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
function enableDiscovery() {
|
||||
originalVitest = process.env.VITEST;
|
||||
originalNodeEnv = process.env.NODE_ENV;
|
||||
originalFetch = globalThis.fetch;
|
||||
delete process.env.VITEST;
|
||||
delete process.env.NODE_ENV;
|
||||
}
|
||||
|
||||
it("does not send marker value as vLLM bearer token during discovery", async () => {
|
||||
enableDiscovery();
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ data: [] }),
|
||||
});
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
|
||||
await writeFile(
|
||||
join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"vllm:default": {
|
||||
type: "api_key",
|
||||
provider: "vllm",
|
||||
keyRef: { source: "file", provider: "vault", id: "/vllm/apiKey" },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const providers = await resolveImplicitProviders({ agentDir });
|
||||
expect(providers?.vllm?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
|
||||
const request = fetchMock.mock.calls[0]?.[1] as
|
||||
| { headers?: Record<string, string> }
|
||||
| undefined;
|
||||
expect(request?.headers?.Authorization).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not call Hugging Face discovery with marker-backed credentials", async () => {
|
||||
enableDiscovery();
|
||||
const fetchMock = vi.fn();
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
|
||||
await writeFile(
|
||||
join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"huggingface:default": {
|
||||
type: "api_key",
|
||||
provider: "huggingface",
|
||||
keyRef: { source: "exec", provider: "vault", id: "providers/hf/token" },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const providers = await resolveImplicitProviders({ agentDir });
|
||||
expect(providers?.huggingface?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
|
||||
const huggingfaceCalls = fetchMock.mock.calls.filter(([url]) =>
|
||||
String(url).includes("router.huggingface.co"),
|
||||
);
|
||||
expect(huggingfaceCalls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@ -1,6 +1,6 @@
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { ModelDefinitionConfig } from "../config/types.models.js";
|
||||
import { coerceSecretRef } from "../config/types.secrets.js";
|
||||
import { coerceSecretRef, resolveSecretInputRef } from "../config/types.secrets.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import {
|
||||
DEFAULT_COPILOT_API_BASE_URL,
|
||||
@ -40,6 +40,13 @@ import {
|
||||
HUGGINGFACE_MODEL_CATALOG,
|
||||
buildHuggingfaceModelDefinition,
|
||||
} from "./huggingface-models.js";
|
||||
import {
|
||||
MINIMAX_OAUTH_MARKER,
|
||||
OLLAMA_LOCAL_AUTH_MARKER,
|
||||
QWEN_OAUTH_MARKER,
|
||||
isNonSecretApiKeyMarker,
|
||||
resolveNonEnvSecretRefApiKeyMarker,
|
||||
} from "./model-auth-markers.js";
|
||||
import { resolveAwsSdkEnvVarName, resolveEnvApiKey } from "./model-auth.js";
|
||||
import { OLLAMA_NATIVE_BASE_URL } from "./ollama-stream.js";
|
||||
import {
|
||||
@ -62,7 +69,6 @@ const MINIMAX_DEFAULT_MODEL_ID = "MiniMax-M2.5";
|
||||
const MINIMAX_DEFAULT_VISION_MODEL_ID = "MiniMax-VL-01";
|
||||
const MINIMAX_DEFAULT_CONTEXT_WINDOW = 200000;
|
||||
const MINIMAX_DEFAULT_MAX_TOKENS = 8192;
|
||||
const MINIMAX_OAUTH_PLACEHOLDER = "minimax-oauth";
|
||||
// Pricing per 1M tokens (USD) — https://platform.minimaxi.com/document/Price
|
||||
const MINIMAX_API_COST = {
|
||||
input: 0.3,
|
||||
@ -132,7 +138,6 @@ const KIMI_CODING_DEFAULT_COST = {
|
||||
};
|
||||
|
||||
const QWEN_PORTAL_BASE_URL = "https://portal.qwen.ai/v1";
|
||||
const QWEN_PORTAL_OAUTH_PLACEHOLDER = "qwen-oauth";
|
||||
const QWEN_PORTAL_DEFAULT_CONTEXT_WINDOW = 128000;
|
||||
const QWEN_PORTAL_DEFAULT_MAX_TOKENS = 8192;
|
||||
const QWEN_PORTAL_DEFAULT_COST = {
|
||||
@ -403,35 +408,88 @@ function resolveAwsSdkApiKeyVarName(): string {
|
||||
return resolveAwsSdkEnvVarName() ?? "AWS_PROFILE";
|
||||
}
|
||||
|
||||
type ProfileApiKeyResolution = {
|
||||
apiKey: string;
|
||||
source: "plaintext" | "env-ref" | "non-env-ref";
|
||||
/** Optional secret value that may be used for provider discovery only. */
|
||||
discoveryApiKey?: string;
|
||||
};
|
||||
|
||||
function toDiscoveryApiKey(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed || isNonSecretApiKeyMarker(trimmed)) {
|
||||
return undefined;
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function resolveApiKeyFromCredential(
|
||||
cred: ReturnType<typeof ensureAuthProfileStore>["profiles"][string] | undefined,
|
||||
): ProfileApiKeyResolution | undefined {
|
||||
if (!cred) {
|
||||
return undefined;
|
||||
}
|
||||
if (cred.type === "api_key") {
|
||||
const keyRef = coerceSecretRef(cred.keyRef);
|
||||
if (keyRef && keyRef.id.trim()) {
|
||||
if (keyRef.source === "env") {
|
||||
const envVar = keyRef.id.trim();
|
||||
return {
|
||||
apiKey: envVar,
|
||||
source: "env-ref",
|
||||
discoveryApiKey: toDiscoveryApiKey(process.env[envVar]),
|
||||
};
|
||||
}
|
||||
return {
|
||||
apiKey: resolveNonEnvSecretRefApiKeyMarker(keyRef.source),
|
||||
source: "non-env-ref",
|
||||
};
|
||||
}
|
||||
if (cred.key?.trim()) {
|
||||
return {
|
||||
apiKey: cred.key,
|
||||
source: "plaintext",
|
||||
discoveryApiKey: toDiscoveryApiKey(cred.key),
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
if (cred.type === "token") {
|
||||
const tokenRef = coerceSecretRef(cred.tokenRef);
|
||||
if (tokenRef && tokenRef.id.trim()) {
|
||||
if (tokenRef.source === "env") {
|
||||
const envVar = tokenRef.id.trim();
|
||||
return {
|
||||
apiKey: envVar,
|
||||
source: "env-ref",
|
||||
discoveryApiKey: toDiscoveryApiKey(process.env[envVar]),
|
||||
};
|
||||
}
|
||||
return {
|
||||
apiKey: resolveNonEnvSecretRefApiKeyMarker(tokenRef.source),
|
||||
source: "non-env-ref",
|
||||
};
|
||||
}
|
||||
if (cred.token?.trim()) {
|
||||
return {
|
||||
apiKey: cred.token,
|
||||
source: "plaintext",
|
||||
discoveryApiKey: toDiscoveryApiKey(cred.token),
|
||||
};
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveApiKeyFromProfiles(params: {
|
||||
provider: string;
|
||||
store: ReturnType<typeof ensureAuthProfileStore>;
|
||||
}): string | undefined {
|
||||
}): ProfileApiKeyResolution | undefined {
|
||||
const ids = listProfilesForProvider(params.store, params.provider);
|
||||
for (const id of ids) {
|
||||
const cred = params.store.profiles[id];
|
||||
if (!cred) {
|
||||
continue;
|
||||
}
|
||||
if (cred.type === "api_key") {
|
||||
if (cred.key?.trim()) {
|
||||
return cred.key;
|
||||
}
|
||||
const keyRef = coerceSecretRef(cred.keyRef);
|
||||
if (keyRef?.source === "env" && keyRef.id.trim()) {
|
||||
return keyRef.id.trim();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (cred.type === "token") {
|
||||
if (cred.token?.trim()) {
|
||||
return cred.token;
|
||||
}
|
||||
const tokenRef = coerceSecretRef(cred.tokenRef);
|
||||
if (tokenRef?.source === "env" && tokenRef.id.trim()) {
|
||||
return tokenRef.id.trim();
|
||||
}
|
||||
continue;
|
||||
const resolved = resolveApiKeyFromCredential(params.store.profiles[id]);
|
||||
if (resolved) {
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
@ -483,6 +541,12 @@ function normalizeAntigravityProvider(provider: ProviderConfig): ProviderConfig
|
||||
export function normalizeProviders(params: {
|
||||
providers: ModelsConfig["providers"];
|
||||
agentDir: string;
|
||||
secretDefaults?: {
|
||||
env?: string;
|
||||
file?: string;
|
||||
exec?: string;
|
||||
};
|
||||
secretRefManagedProviders?: Set<string>;
|
||||
}): ModelsConfig["providers"] {
|
||||
const { providers } = params;
|
||||
if (!providers) {
|
||||
@ -505,17 +569,42 @@ export function normalizeProviders(params: {
|
||||
}
|
||||
let normalizedProvider = provider;
|
||||
const configuredApiKey = normalizedProvider.apiKey;
|
||||
const configuredApiKeyRef = resolveSecretInputRef({
|
||||
value: configuredApiKey,
|
||||
defaults: params.secretDefaults,
|
||||
}).ref;
|
||||
const profileApiKey = resolveApiKeyFromProfiles({
|
||||
provider: normalizedKey,
|
||||
store: authStore,
|
||||
});
|
||||
|
||||
// Fix common misconfig: apiKey set to "${ENV_VAR}" instead of "ENV_VAR".
|
||||
if (
|
||||
typeof configuredApiKey === "string" &&
|
||||
normalizeApiKeyConfig(configuredApiKey) !== configuredApiKey
|
||||
) {
|
||||
mutated = true;
|
||||
normalizedProvider = {
|
||||
...normalizedProvider,
|
||||
apiKey: normalizeApiKeyConfig(configuredApiKey),
|
||||
};
|
||||
if (configuredApiKeyRef && configuredApiKeyRef.id.trim()) {
|
||||
const marker =
|
||||
configuredApiKeyRef.source === "env"
|
||||
? configuredApiKeyRef.id.trim()
|
||||
: resolveNonEnvSecretRefApiKeyMarker(configuredApiKeyRef.source);
|
||||
if (normalizedProvider.apiKey !== marker) {
|
||||
mutated = true;
|
||||
normalizedProvider = { ...normalizedProvider, apiKey: marker };
|
||||
}
|
||||
params.secretRefManagedProviders?.add(normalizedKey);
|
||||
} else if (typeof configuredApiKey === "string") {
|
||||
// Fix common misconfig: apiKey set to "${ENV_VAR}" instead of "ENV_VAR".
|
||||
const normalizedConfiguredApiKey = normalizeApiKeyConfig(configuredApiKey);
|
||||
if (normalizedConfiguredApiKey !== configuredApiKey) {
|
||||
mutated = true;
|
||||
normalizedProvider = {
|
||||
...normalizedProvider,
|
||||
apiKey: normalizedConfiguredApiKey,
|
||||
};
|
||||
}
|
||||
if (
|
||||
profileApiKey &&
|
||||
profileApiKey.source !== "plaintext" &&
|
||||
normalizedConfiguredApiKey === profileApiKey.apiKey
|
||||
) {
|
||||
params.secretRefManagedProviders?.add(normalizedKey);
|
||||
}
|
||||
}
|
||||
|
||||
// If a provider defines models, pi's ModelRegistry requires apiKey to be set.
|
||||
@ -533,12 +622,11 @@ export function normalizeProviders(params: {
|
||||
normalizedProvider = { ...normalizedProvider, apiKey };
|
||||
} else {
|
||||
const fromEnv = resolveEnvApiKeyVarName(normalizedKey);
|
||||
const fromProfiles = resolveApiKeyFromProfiles({
|
||||
provider: normalizedKey,
|
||||
store: authStore,
|
||||
});
|
||||
const apiKey = fromEnv ?? fromProfiles;
|
||||
const apiKey = fromEnv ?? profileApiKey?.apiKey;
|
||||
if (apiKey?.trim()) {
|
||||
if (profileApiKey && profileApiKey.source !== "plaintext") {
|
||||
params.secretRefManagedProviders?.add(normalizedKey);
|
||||
}
|
||||
mutated = true;
|
||||
normalizedProvider = { ...normalizedProvider, apiKey };
|
||||
}
|
||||
@ -777,14 +865,8 @@ async function buildOllamaProvider(
|
||||
};
|
||||
}
|
||||
|
||||
async function buildHuggingfaceProvider(apiKey?: string): Promise<ProviderConfig> {
|
||||
// Resolve env var name to value for discovery (GET /v1/models requires Bearer token).
|
||||
const resolvedSecret =
|
||||
apiKey?.trim() !== ""
|
||||
? /^[A-Z][A-Z0-9_]*$/.test(apiKey!.trim())
|
||||
? (process.env[apiKey!.trim()] ?? "").trim()
|
||||
: apiKey!.trim()
|
||||
: "";
|
||||
async function buildHuggingfaceProvider(discoveryApiKey?: string): Promise<ProviderConfig> {
|
||||
const resolvedSecret = toDiscoveryApiKey(discoveryApiKey) ?? "";
|
||||
const models =
|
||||
resolvedSecret !== ""
|
||||
? await discoverHuggingfaceModels(resolvedSecret)
|
||||
@ -928,10 +1010,24 @@ export async function resolveImplicitProviders(params: {
|
||||
const authStore = ensureAuthProfileStore(params.agentDir, {
|
||||
allowKeychainPrompt: false,
|
||||
});
|
||||
const resolveProviderApiKey = (
|
||||
provider: string,
|
||||
): { apiKey: string | undefined; discoveryApiKey?: string } => {
|
||||
const envVar = resolveEnvApiKeyVarName(provider);
|
||||
if (envVar) {
|
||||
return {
|
||||
apiKey: envVar,
|
||||
discoveryApiKey: toDiscoveryApiKey(process.env[envVar]),
|
||||
};
|
||||
}
|
||||
const fromProfiles = resolveApiKeyFromProfiles({ provider, store: authStore });
|
||||
return {
|
||||
apiKey: fromProfiles?.apiKey,
|
||||
discoveryApiKey: fromProfiles?.discoveryApiKey,
|
||||
};
|
||||
};
|
||||
|
||||
const minimaxKey =
|
||||
resolveEnvApiKeyVarName("minimax") ??
|
||||
resolveApiKeyFromProfiles({ provider: "minimax", store: authStore });
|
||||
const minimaxKey = resolveProviderApiKey("minimax").apiKey;
|
||||
if (minimaxKey) {
|
||||
providers.minimax = { ...buildMinimaxProvider(), apiKey: minimaxKey };
|
||||
}
|
||||
@ -940,34 +1036,26 @@ export async function resolveImplicitProviders(params: {
|
||||
if (minimaxOauthProfile.length > 0) {
|
||||
providers["minimax-portal"] = {
|
||||
...buildMinimaxPortalProvider(),
|
||||
apiKey: MINIMAX_OAUTH_PLACEHOLDER,
|
||||
apiKey: MINIMAX_OAUTH_MARKER,
|
||||
};
|
||||
}
|
||||
|
||||
const moonshotKey =
|
||||
resolveEnvApiKeyVarName("moonshot") ??
|
||||
resolveApiKeyFromProfiles({ provider: "moonshot", store: authStore });
|
||||
const moonshotKey = resolveProviderApiKey("moonshot").apiKey;
|
||||
if (moonshotKey) {
|
||||
providers.moonshot = { ...buildMoonshotProvider(), apiKey: moonshotKey };
|
||||
}
|
||||
|
||||
const kimiCodingKey =
|
||||
resolveEnvApiKeyVarName("kimi-coding") ??
|
||||
resolveApiKeyFromProfiles({ provider: "kimi-coding", store: authStore });
|
||||
const kimiCodingKey = resolveProviderApiKey("kimi-coding").apiKey;
|
||||
if (kimiCodingKey) {
|
||||
providers["kimi-coding"] = { ...buildKimiCodingProvider(), apiKey: kimiCodingKey };
|
||||
}
|
||||
|
||||
const syntheticKey =
|
||||
resolveEnvApiKeyVarName("synthetic") ??
|
||||
resolveApiKeyFromProfiles({ provider: "synthetic", store: authStore });
|
||||
const syntheticKey = resolveProviderApiKey("synthetic").apiKey;
|
||||
if (syntheticKey) {
|
||||
providers.synthetic = { ...buildSyntheticProvider(), apiKey: syntheticKey };
|
||||
}
|
||||
|
||||
const veniceKey =
|
||||
resolveEnvApiKeyVarName("venice") ??
|
||||
resolveApiKeyFromProfiles({ provider: "venice", store: authStore });
|
||||
const veniceKey = resolveProviderApiKey("venice").apiKey;
|
||||
if (veniceKey) {
|
||||
providers.venice = { ...(await buildVeniceProvider()), apiKey: veniceKey };
|
||||
}
|
||||
@ -976,13 +1064,11 @@ export async function resolveImplicitProviders(params: {
|
||||
if (qwenProfiles.length > 0) {
|
||||
providers["qwen-portal"] = {
|
||||
...buildQwenPortalProvider(),
|
||||
apiKey: QWEN_PORTAL_OAUTH_PLACEHOLDER,
|
||||
apiKey: QWEN_OAUTH_MARKER,
|
||||
};
|
||||
}
|
||||
|
||||
const volcengineKey =
|
||||
resolveEnvApiKeyVarName("volcengine") ??
|
||||
resolveApiKeyFromProfiles({ provider: "volcengine", store: authStore });
|
||||
const volcengineKey = resolveProviderApiKey("volcengine").apiKey;
|
||||
if (volcengineKey) {
|
||||
providers.volcengine = { ...buildDoubaoProvider(), apiKey: volcengineKey };
|
||||
providers["volcengine-plan"] = {
|
||||
@ -991,9 +1077,7 @@ export async function resolveImplicitProviders(params: {
|
||||
};
|
||||
}
|
||||
|
||||
const byteplusKey =
|
||||
resolveEnvApiKeyVarName("byteplus") ??
|
||||
resolveApiKeyFromProfiles({ provider: "byteplus", store: authStore });
|
||||
const byteplusKey = resolveProviderApiKey("byteplus").apiKey;
|
||||
if (byteplusKey) {
|
||||
providers.byteplus = { ...buildBytePlusProvider(), apiKey: byteplusKey };
|
||||
providers["byteplus-plan"] = {
|
||||
@ -1002,9 +1086,7 @@ export async function resolveImplicitProviders(params: {
|
||||
};
|
||||
}
|
||||
|
||||
const xiaomiKey =
|
||||
resolveEnvApiKeyVarName("xiaomi") ??
|
||||
resolveApiKeyFromProfiles({ provider: "xiaomi", store: authStore });
|
||||
const xiaomiKey = resolveProviderApiKey("xiaomi").apiKey;
|
||||
if (xiaomiKey) {
|
||||
providers.xiaomi = { ...buildXiaomiProvider(), apiKey: xiaomiKey };
|
||||
}
|
||||
@ -1024,7 +1106,9 @@ export async function resolveImplicitProviders(params: {
|
||||
if (!baseUrl) {
|
||||
continue;
|
||||
}
|
||||
const apiKey = resolveEnvApiKeyVarName("cloudflare-ai-gateway") ?? cred.key?.trim() ?? "";
|
||||
const envVarApiKey = resolveEnvApiKeyVarName("cloudflare-ai-gateway");
|
||||
const profileApiKey = resolveApiKeyFromCredential(cred)?.apiKey;
|
||||
const apiKey = envVarApiKey ?? profileApiKey ?? "";
|
||||
if (!apiKey) {
|
||||
continue;
|
||||
}
|
||||
@ -1041,9 +1125,7 @@ export async function resolveImplicitProviders(params: {
|
||||
// Use the user's configured baseUrl (from explicit providers) for model
|
||||
// discovery so that remote / non-default Ollama instances are reachable.
|
||||
// Skip discovery when explicit models are already defined.
|
||||
const ollamaKey =
|
||||
resolveEnvApiKeyVarName("ollama") ??
|
||||
resolveApiKeyFromProfiles({ provider: "ollama", store: authStore });
|
||||
const ollamaKey = resolveProviderApiKey("ollama").apiKey;
|
||||
const explicitOllama = params.explicitProviders?.ollama;
|
||||
const hasExplicitModels =
|
||||
Array.isArray(explicitOllama?.models) && explicitOllama.models.length > 0;
|
||||
@ -1052,7 +1134,7 @@ export async function resolveImplicitProviders(params: {
|
||||
...explicitOllama,
|
||||
baseUrl: resolveOllamaApiBase(explicitOllama.baseUrl),
|
||||
api: explicitOllama.api ?? "ollama",
|
||||
apiKey: ollamaKey ?? explicitOllama.apiKey ?? "ollama-local",
|
||||
apiKey: ollamaKey ?? explicitOllama.apiKey ?? OLLAMA_LOCAL_AUTH_MARKER,
|
||||
};
|
||||
} else {
|
||||
const ollamaBaseUrl = explicitOllama?.baseUrl;
|
||||
@ -1065,7 +1147,7 @@ export async function resolveImplicitProviders(params: {
|
||||
if (ollamaProvider.models.length > 0 || ollamaKey || explicitOllama?.apiKey) {
|
||||
providers.ollama = {
|
||||
...ollamaProvider,
|
||||
apiKey: ollamaKey ?? explicitOllama?.apiKey ?? "ollama-local",
|
||||
apiKey: ollamaKey ?? explicitOllama?.apiKey ?? OLLAMA_LOCAL_AUTH_MARKER,
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -1073,23 +1155,16 @@ export async function resolveImplicitProviders(params: {
|
||||
// vLLM provider - OpenAI-compatible local server (opt-in via env/profile).
|
||||
// If explicitly configured, keep user-defined models/settings as-is.
|
||||
if (!params.explicitProviders?.vllm) {
|
||||
const vllmEnvVar = resolveEnvApiKeyVarName("vllm");
|
||||
const vllmProfileKey = resolveApiKeyFromProfiles({ provider: "vllm", store: authStore });
|
||||
const vllmKey = vllmEnvVar ?? vllmProfileKey;
|
||||
const { apiKey: vllmKey, discoveryApiKey } = resolveProviderApiKey("vllm");
|
||||
if (vllmKey) {
|
||||
const discoveryApiKey = vllmEnvVar
|
||||
? (process.env[vllmEnvVar]?.trim() ?? "")
|
||||
: (vllmProfileKey ?? "");
|
||||
providers.vllm = {
|
||||
...(await buildVllmProvider({ apiKey: discoveryApiKey || undefined })),
|
||||
...(await buildVllmProvider({ apiKey: discoveryApiKey })),
|
||||
apiKey: vllmKey,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const togetherKey =
|
||||
resolveEnvApiKeyVarName("together") ??
|
||||
resolveApiKeyFromProfiles({ provider: "together", store: authStore });
|
||||
const togetherKey = resolveProviderApiKey("together").apiKey;
|
||||
if (togetherKey) {
|
||||
providers.together = {
|
||||
...buildTogetherProvider(),
|
||||
@ -1097,41 +1172,32 @@ export async function resolveImplicitProviders(params: {
|
||||
};
|
||||
}
|
||||
|
||||
const huggingfaceKey =
|
||||
resolveEnvApiKeyVarName("huggingface") ??
|
||||
resolveApiKeyFromProfiles({ provider: "huggingface", store: authStore });
|
||||
const { apiKey: huggingfaceKey, discoveryApiKey: huggingfaceDiscoveryApiKey } =
|
||||
resolveProviderApiKey("huggingface");
|
||||
if (huggingfaceKey) {
|
||||
const hfProvider = await buildHuggingfaceProvider(huggingfaceKey);
|
||||
const hfProvider = await buildHuggingfaceProvider(huggingfaceDiscoveryApiKey);
|
||||
providers.huggingface = {
|
||||
...hfProvider,
|
||||
apiKey: huggingfaceKey,
|
||||
};
|
||||
}
|
||||
|
||||
const qianfanKey =
|
||||
resolveEnvApiKeyVarName("qianfan") ??
|
||||
resolveApiKeyFromProfiles({ provider: "qianfan", store: authStore });
|
||||
const qianfanKey = resolveProviderApiKey("qianfan").apiKey;
|
||||
if (qianfanKey) {
|
||||
providers.qianfan = { ...buildQianfanProvider(), apiKey: qianfanKey };
|
||||
}
|
||||
|
||||
const openrouterKey =
|
||||
resolveEnvApiKeyVarName("openrouter") ??
|
||||
resolveApiKeyFromProfiles({ provider: "openrouter", store: authStore });
|
||||
const openrouterKey = resolveProviderApiKey("openrouter").apiKey;
|
||||
if (openrouterKey) {
|
||||
providers.openrouter = { ...buildOpenrouterProvider(), apiKey: openrouterKey };
|
||||
}
|
||||
|
||||
const nvidiaKey =
|
||||
resolveEnvApiKeyVarName("nvidia") ??
|
||||
resolveApiKeyFromProfiles({ provider: "nvidia", store: authStore });
|
||||
const nvidiaKey = resolveProviderApiKey("nvidia").apiKey;
|
||||
if (nvidiaKey) {
|
||||
providers.nvidia = { ...buildNvidiaProvider(), apiKey: nvidiaKey };
|
||||
}
|
||||
|
||||
const kilocodeKey =
|
||||
resolveEnvApiKeyVarName("kilocode") ??
|
||||
resolveApiKeyFromProfiles({ provider: "kilocode", store: authStore });
|
||||
const kilocodeKey = resolveProviderApiKey("kilocode").apiKey;
|
||||
if (kilocodeKey) {
|
||||
providers.kilocode = { ...buildKilocodeProvider(), apiKey: kilocodeKey };
|
||||
}
|
||||
|
||||
103
src/agents/models-config.runtime-source-snapshot.test.ts
Normal file
103
src/agents/models-config.runtime-source-snapshot.test.ts
Normal file
@ -0,0 +1,103 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import {
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
loadConfig,
|
||||
setRuntimeConfigSnapshot,
|
||||
} from "../config/config.js";
|
||||
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
|
||||
import {
|
||||
installModelsConfigTestHooks,
|
||||
withModelsTempHome as withTempHome,
|
||||
} from "./models-config.e2e-harness.js";
|
||||
import { ensureOpenClawModelsJson } from "./models-config.js";
|
||||
import { readGeneratedModelsJson } from "./models-config.test-utils.js";
|
||||
|
||||
installModelsConfigTestHooks();
|
||||
|
||||
describe("models-config runtime source snapshot", () => {
|
||||
it("uses runtime source snapshot markers when passed the active runtime config", async () => {
|
||||
await withTempHome(async () => {
|
||||
const sourceConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
|
||||
api: "openai-completions" as const,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const runtimeConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: "sk-runtime-resolved",
|
||||
api: "openai-completions" as const,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
|
||||
await ensureOpenClawModelsJson(loadConfig());
|
||||
|
||||
const parsed = await readGeneratedModelsJson<{
|
||||
providers: Record<string, { apiKey?: string }>;
|
||||
}>();
|
||||
expect(parsed.providers.openai?.apiKey).toBe("OPENAI_API_KEY");
|
||||
} finally {
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("uses non-env marker from runtime source snapshot for file refs", async () => {
|
||||
await withTempHome(async () => {
|
||||
const sourceConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
moonshot: {
|
||||
baseUrl: "https://api.moonshot.ai/v1",
|
||||
apiKey: { source: "file", provider: "vault", id: "/moonshot/apiKey" },
|
||||
api: "openai-completions" as const,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const runtimeConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
moonshot: {
|
||||
baseUrl: "https://api.moonshot.ai/v1",
|
||||
apiKey: "sk-runtime-moonshot",
|
||||
api: "openai-completions" as const,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
|
||||
await ensureOpenClawModelsJson(loadConfig());
|
||||
|
||||
const parsed = await readGeneratedModelsJson<{
|
||||
providers: Record<string, { apiKey?: string }>;
|
||||
}>();
|
||||
expect(parsed.providers.moonshot?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
|
||||
} finally {
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,6 +1,11 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { type OpenClawConfig, loadConfig } from "../config/config.js";
|
||||
import {
|
||||
getRuntimeConfigSnapshot,
|
||||
getRuntimeConfigSourceSnapshot,
|
||||
type OpenClawConfig,
|
||||
loadConfig,
|
||||
} from "../config/config.js";
|
||||
import { applyConfigEnvVars } from "../config/env-vars.js";
|
||||
import { isRecord } from "../utils.js";
|
||||
import { resolveOpenClawAgentDir } from "./agent-paths.js";
|
||||
@ -141,8 +146,9 @@ async function resolveProvidersForModelsJson(params: {
|
||||
function mergeWithExistingProviderSecrets(params: {
|
||||
nextProviders: Record<string, ProviderConfig>;
|
||||
existingProviders: Record<string, NonNullable<ModelsConfig["providers"]>[string]>;
|
||||
secretRefManagedProviders: ReadonlySet<string>;
|
||||
}): Record<string, ProviderConfig> {
|
||||
const { nextProviders, existingProviders } = params;
|
||||
const { nextProviders, existingProviders, secretRefManagedProviders } = params;
|
||||
const mergedProviders: Record<string, ProviderConfig> = {};
|
||||
for (const [key, entry] of Object.entries(existingProviders)) {
|
||||
mergedProviders[key] = entry;
|
||||
@ -159,7 +165,11 @@ function mergeWithExistingProviderSecrets(params: {
|
||||
continue;
|
||||
}
|
||||
const preserved: Record<string, unknown> = {};
|
||||
if (typeof existing.apiKey === "string" && existing.apiKey) {
|
||||
if (
|
||||
!secretRefManagedProviders.has(key) &&
|
||||
typeof existing.apiKey === "string" &&
|
||||
existing.apiKey
|
||||
) {
|
||||
preserved.apiKey = existing.apiKey;
|
||||
}
|
||||
if (typeof existing.baseUrl === "string" && existing.baseUrl) {
|
||||
@ -174,6 +184,7 @@ async function resolveProvidersForMode(params: {
|
||||
mode: NonNullable<ModelsConfig["mode"]>;
|
||||
targetPath: string;
|
||||
providers: Record<string, ProviderConfig>;
|
||||
secretRefManagedProviders: ReadonlySet<string>;
|
||||
}): Promise<Record<string, ProviderConfig>> {
|
||||
if (params.mode !== "merge") {
|
||||
return params.providers;
|
||||
@ -189,6 +200,7 @@ async function resolveProvidersForMode(params: {
|
||||
return mergeWithExistingProviderSecrets({
|
||||
nextProviders: params.providers,
|
||||
existingProviders,
|
||||
secretRefManagedProviders: params.secretRefManagedProviders,
|
||||
});
|
||||
}
|
||||
|
||||
@ -200,11 +212,32 @@ async function readRawFile(pathname: string): Promise<string> {
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureModelsFileMode(pathname: string): Promise<void> {
|
||||
await fs.chmod(pathname, 0o600).catch(() => {
|
||||
// best-effort
|
||||
});
|
||||
}
|
||||
|
||||
function resolveModelsConfigInput(config?: OpenClawConfig): OpenClawConfig {
|
||||
const runtimeSource = getRuntimeConfigSourceSnapshot();
|
||||
if (!runtimeSource) {
|
||||
return config ?? loadConfig();
|
||||
}
|
||||
if (!config) {
|
||||
return runtimeSource;
|
||||
}
|
||||
const runtimeResolved = getRuntimeConfigSnapshot();
|
||||
if (runtimeResolved && config === runtimeResolved) {
|
||||
return runtimeSource;
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
export async function ensureOpenClawModelsJson(
|
||||
config?: OpenClawConfig,
|
||||
agentDirOverride?: string,
|
||||
): Promise<{ agentDir: string; wrote: boolean }> {
|
||||
const cfg = config ?? loadConfig();
|
||||
const cfg = resolveModelsConfigInput(config);
|
||||
const agentDir = agentDirOverride?.trim() ? agentDirOverride.trim() : resolveOpenClawAgentDir();
|
||||
|
||||
// Ensure config env vars (e.g. AWS_PROFILE, AWS_ACCESS_KEY_ID) are
|
||||
@ -221,24 +254,31 @@ export async function ensureOpenClawModelsJson(
|
||||
|
||||
const mode = cfg.models?.mode ?? DEFAULT_MODE;
|
||||
const targetPath = path.join(agentDir, "models.json");
|
||||
const secretRefManagedProviders = new Set<string>();
|
||||
|
||||
const normalizedProviders =
|
||||
normalizeProviders({
|
||||
providers,
|
||||
agentDir,
|
||||
secretDefaults: cfg.secrets?.defaults,
|
||||
secretRefManagedProviders,
|
||||
}) ?? providers;
|
||||
const mergedProviders = await resolveProvidersForMode({
|
||||
mode,
|
||||
targetPath,
|
||||
providers,
|
||||
providers: normalizedProviders,
|
||||
secretRefManagedProviders,
|
||||
});
|
||||
|
||||
const normalizedProviders = normalizeProviders({
|
||||
providers: mergedProviders,
|
||||
agentDir,
|
||||
});
|
||||
const next = `${JSON.stringify({ providers: normalizedProviders }, null, 2)}\n`;
|
||||
const next = `${JSON.stringify({ providers: mergedProviders }, null, 2)}\n`;
|
||||
const existingRaw = await readRawFile(targetPath);
|
||||
|
||||
if (existingRaw === next) {
|
||||
await ensureModelsFileMode(targetPath);
|
||||
return { agentDir, wrote: false };
|
||||
}
|
||||
|
||||
await fs.mkdir(agentDir, { recursive: true, mode: 0o700 });
|
||||
await fs.writeFile(targetPath, next, { mode: 0o600 });
|
||||
await ensureModelsFileMode(targetPath);
|
||||
return { agentDir, wrote: true };
|
||||
}
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { NON_ENV_SECRETREF_MARKER } from "../../agents/model-auth-markers.js";
|
||||
import { resolveProviderAuthOverview } from "./list.auth-overview.js";
|
||||
|
||||
describe("resolveProviderAuthOverview", () => {
|
||||
@ -21,4 +22,52 @@ describe("resolveProviderAuthOverview", () => {
|
||||
|
||||
expect(overview.profiles.labels[0]).toContain("token:ref(env:GITHUB_TOKEN)");
|
||||
});
|
||||
|
||||
it("renders marker-backed models.json auth as marker detail", () => {
|
||||
const overview = resolveProviderAuthOverview({
|
||||
provider: "openai",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: NON_ENV_SECRETREF_MARKER,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
store: { version: 1, profiles: {} } as never,
|
||||
modelsPath: "/tmp/models.json",
|
||||
});
|
||||
|
||||
expect(overview.effective.kind).toBe("models.json");
|
||||
expect(overview.effective.detail).toContain(`marker(${NON_ENV_SECRETREF_MARKER})`);
|
||||
expect(overview.modelsJson?.value).toContain(`marker(${NON_ENV_SECRETREF_MARKER})`);
|
||||
});
|
||||
|
||||
it("keeps env-var-shaped models.json values masked to avoid accidental plaintext exposure", () => {
|
||||
const overview = resolveProviderAuthOverview({
|
||||
provider: "openai",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "OPENAI_API_KEY",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
store: { version: 1, profiles: {} } as never,
|
||||
modelsPath: "/tmp/models.json",
|
||||
});
|
||||
|
||||
expect(overview.effective.kind).toBe("models.json");
|
||||
expect(overview.effective.detail).not.toContain("marker(");
|
||||
expect(overview.effective.detail).not.toContain("OPENAI_API_KEY");
|
||||
});
|
||||
});
|
||||
|
||||
@ -6,12 +6,19 @@ import {
|
||||
resolveAuthStorePathForDisplay,
|
||||
resolveProfileUnusableUntilForDisplay,
|
||||
} from "../../agents/auth-profiles.js";
|
||||
import { isNonSecretApiKeyMarker } from "../../agents/model-auth-markers.js";
|
||||
import { getCustomProviderApiKey, resolveEnvApiKey } from "../../agents/model-auth.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { shortenHomePath } from "../../utils.js";
|
||||
import { maskApiKey } from "./list.format.js";
|
||||
import type { ProviderAuthOverview } from "./list.types.js";
|
||||
|
||||
function formatMarkerOrSecret(value: string): string {
|
||||
return isNonSecretApiKeyMarker(value, { includeEnvVarName: false })
|
||||
? `marker(${value.trim()})`
|
||||
: maskApiKey(value);
|
||||
}
|
||||
|
||||
function formatProfileSecretLabel(params: {
|
||||
value: string | undefined;
|
||||
ref: { source: string; id: string } | undefined;
|
||||
@ -19,7 +26,8 @@ function formatProfileSecretLabel(params: {
|
||||
}): string {
|
||||
const value = typeof params.value === "string" ? params.value.trim() : "";
|
||||
if (value) {
|
||||
return params.kind === "token" ? `token:${maskApiKey(value)}` : maskApiKey(value);
|
||||
const display = formatMarkerOrSecret(value);
|
||||
return params.kind === "token" ? `token:${display}` : display;
|
||||
}
|
||||
if (params.ref) {
|
||||
const refLabel = `ref(${params.ref.source}:${params.ref.id})`;
|
||||
@ -108,7 +116,7 @@ export function resolveProviderAuthOverview(params: {
|
||||
};
|
||||
}
|
||||
if (customKey) {
|
||||
return { kind: "models.json", detail: maskApiKey(customKey) };
|
||||
return { kind: "models.json", detail: formatMarkerOrSecret(customKey) };
|
||||
}
|
||||
return { kind: "missing", detail: "missing" };
|
||||
})();
|
||||
@ -137,7 +145,7 @@ export function resolveProviderAuthOverview(params: {
|
||||
...(customKey
|
||||
? {
|
||||
modelsJson: {
|
||||
value: maskApiKey(customKey),
|
||||
value: formatMarkerOrSecret(customKey),
|
||||
source: `models.json: ${shortenHomePath(params.modelsPath)}`,
|
||||
},
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AuthProfileStore } from "../../agents/auth-profiles.js";
|
||||
import { OLLAMA_LOCAL_AUTH_MARKER } from "../../agents/model-auth-markers.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
|
||||
let mockStore: AuthProfileStore;
|
||||
@ -163,4 +164,53 @@ describe("buildProbeTargets reason codes", () => {
|
||||
expect(plan.results[0]?.error).toContain("[unresolved_ref]");
|
||||
expect(plan.results[0]?.error).toContain("env:default:MISSING_ANTHROPIC_TOKEN");
|
||||
});
|
||||
|
||||
it("skips marker-only models.json credentials when building probe targets", async () => {
|
||||
const previousAnthropic = process.env.ANTHROPIC_API_KEY;
|
||||
const previousAnthropicOauth = process.env.ANTHROPIC_OAUTH_TOKEN;
|
||||
delete process.env.ANTHROPIC_API_KEY;
|
||||
delete process.env.ANTHROPIC_OAUTH_TOKEN;
|
||||
mockStore = {
|
||||
version: 1,
|
||||
profiles: {},
|
||||
order: {},
|
||||
};
|
||||
try {
|
||||
const plan = await buildProbeTargets({
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
anthropic: {
|
||||
baseUrl: "https://api.anthropic.com/v1",
|
||||
api: "anthropic-messages",
|
||||
apiKey: OLLAMA_LOCAL_AUTH_MARKER,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
providers: ["anthropic"],
|
||||
modelCandidates: ["anthropic/claude-sonnet-4-6"],
|
||||
options: {
|
||||
timeoutMs: 5_000,
|
||||
concurrency: 1,
|
||||
maxTokens: 16,
|
||||
},
|
||||
});
|
||||
|
||||
expect(plan.targets).toEqual([]);
|
||||
expect(plan.results).toEqual([]);
|
||||
} finally {
|
||||
if (previousAnthropic === undefined) {
|
||||
delete process.env.ANTHROPIC_API_KEY;
|
||||
} else {
|
||||
process.env.ANTHROPIC_API_KEY = previousAnthropic;
|
||||
}
|
||||
if (previousAnthropicOauth === undefined) {
|
||||
delete process.env.ANTHROPIC_OAUTH_TOKEN;
|
||||
} else {
|
||||
process.env.ANTHROPIC_OAUTH_TOKEN = previousAnthropicOauth;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@ -12,6 +12,7 @@ import {
|
||||
resolveAuthProfileOrder,
|
||||
} from "../../agents/auth-profiles.js";
|
||||
import { describeFailoverError } from "../../agents/failover-error.js";
|
||||
import { isNonSecretApiKeyMarker } from "../../agents/model-auth-markers.js";
|
||||
import { getCustomProviderApiKey, resolveEnvApiKey } from "../../agents/model-auth.js";
|
||||
import { loadModelCatalog } from "../../agents/model-catalog.js";
|
||||
import {
|
||||
@ -373,7 +374,8 @@ export async function buildProbeTargets(params: {
|
||||
|
||||
const envKey = resolveEnvApiKey(providerKey);
|
||||
const customKey = getCustomProviderApiKey(cfg, providerKey);
|
||||
if (!envKey && !customKey) {
|
||||
const hasUsableModelsJsonKey = Boolean(customKey && !isNonSecretApiKeyMarker(customKey));
|
||||
if (!envKey && !hasUsableModelsJsonKey) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ export {
|
||||
clearRuntimeConfigSnapshot,
|
||||
createConfigIO,
|
||||
getRuntimeConfigSnapshot,
|
||||
getRuntimeConfigSourceSnapshot,
|
||||
loadConfig,
|
||||
parseConfigJson5,
|
||||
readConfigFileSnapshot,
|
||||
|
||||
@ -5,6 +5,7 @@ import { withTempHome } from "./home-env.test-harness.js";
|
||||
import {
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
getRuntimeConfigSourceSnapshot,
|
||||
loadConfig,
|
||||
setRuntimeConfigSnapshot,
|
||||
writeConfigFile,
|
||||
@ -12,6 +13,70 @@ import {
|
||||
import type { OpenClawConfig } from "./types.js";
|
||||
|
||||
describe("runtime config snapshot writes", () => {
|
||||
it("returns the source snapshot when runtime snapshot is active", async () => {
|
||||
await withTempHome("openclaw-config-runtime-source-", async () => {
|
||||
const sourceConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const runtimeConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: "sk-runtime-resolved",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
try {
|
||||
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
|
||||
expect(getRuntimeConfigSourceSnapshot()).toEqual(sourceConfig);
|
||||
} finally {
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("clears runtime source snapshot when runtime snapshot is cleared", async () => {
|
||||
const sourceConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const runtimeConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: "sk-runtime-resolved",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
expect(getRuntimeConfigSourceSnapshot()).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves source secret refs when writeConfigFile receives runtime-resolved config", async () => {
|
||||
await withTempHome("openclaw-config-runtime-write-", async (home) => {
|
||||
const configPath = path.join(home, ".openclaw", "openclaw.json");
|
||||
|
||||
@ -1345,6 +1345,10 @@ export function getRuntimeConfigSnapshot(): OpenClawConfig | null {
|
||||
return runtimeConfigSnapshot;
|
||||
}
|
||||
|
||||
export function getRuntimeConfigSourceSnapshot(): OpenClawConfig | null {
|
||||
return runtimeConfigSourceSnapshot;
|
||||
}
|
||||
|
||||
export function loadConfig(): OpenClawConfig {
|
||||
if (runtimeConfigSnapshot) {
|
||||
return runtimeConfigSnapshot;
|
||||
|
||||
@ -774,6 +774,9 @@ describe("config help copy quality", () => {
|
||||
it("documents auth/model root semantics and provider secret handling", () => {
|
||||
const providerKey = FIELD_HELP["models.providers.*.apiKey"];
|
||||
expect(/secret|env|credential/i.test(providerKey)).toBe(true);
|
||||
const modelsMode = FIELD_HELP["models.mode"];
|
||||
expect(modelsMode.includes("SecretRef-managed")).toBe(true);
|
||||
expect(modelsMode.includes("preserve")).toBe(true);
|
||||
|
||||
const bedrockRefresh = FIELD_HELP["models.bedrockDiscovery.refreshInterval"];
|
||||
expect(/refresh|seconds|interval/i.test(bedrockRefresh)).toBe(true);
|
||||
|
||||
@ -688,7 +688,7 @@ export const FIELD_HELP: Record<string, string> = {
|
||||
models:
|
||||
"Model catalog root for provider definitions, merge/replace behavior, and optional Bedrock discovery integration. Keep provider definitions explicit and validated before relying on production failover paths.",
|
||||
"models.mode":
|
||||
'Controls provider catalog behavior: "merge" keeps built-ins and overlays your custom providers, while "replace" uses only your configured providers. In "merge", matching provider IDs preserve non-empty agent models.json apiKey/baseUrl values and fall back to config when agent values are empty or missing; matching model contextWindow/maxTokens use the higher value between explicit and implicit entries.',
|
||||
'Controls provider catalog behavior: "merge" keeps built-ins and overlays your custom providers, while "replace" uses only your configured providers. In "merge", matching provider IDs preserve non-empty agent models.json apiKey/baseUrl values only when the provider is not SecretRef-managed in current config/auth-profile context; SecretRef-managed providers refresh apiKey from current source markers while matching model contextWindow/maxTokens use the higher value between explicit and implicit entries.',
|
||||
"models.providers":
|
||||
"Provider map keyed by provider ID containing connection/auth settings and concrete model definitions. Use stable provider keys so references from agents and tooling remain portable across environments.",
|
||||
"models.providers.*.baseUrl":
|
||||
|
||||
@ -2,6 +2,7 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { NON_ENV_SECRETREF_MARKER } from "../agents/model-auth-markers.js";
|
||||
import { resolveProviderAuths } from "./provider-usage.auth.js";
|
||||
|
||||
describe("resolveProviderAuths key normalization", () => {
|
||||
@ -403,4 +404,40 @@ describe("resolveProviderAuths key normalization", () => {
|
||||
expect(auths).toEqual([{ provider: "anthropic", token: "token-1" }]);
|
||||
}, {});
|
||||
});
|
||||
|
||||
it("ignores marker-backed config keys for provider usage auth resolution", async () => {
|
||||
await withSuiteHome(
|
||||
async (home) => {
|
||||
const modelDef = {
|
||||
id: "test-model",
|
||||
name: "Test Model",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1024,
|
||||
maxTokens: 256,
|
||||
};
|
||||
await writeConfig(home, {
|
||||
models: {
|
||||
providers: {
|
||||
minimax: {
|
||||
baseUrl: "https://api.minimaxi.com",
|
||||
models: [modelDef],
|
||||
apiKey: NON_ENV_SECRETREF_MARKER,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const auths = await resolveProviderAuths({
|
||||
providers: ["minimax"],
|
||||
});
|
||||
expect(auths).toEqual([]);
|
||||
},
|
||||
{
|
||||
MINIMAX_API_KEY: undefined,
|
||||
MINIMAX_CODE_PLAN_KEY: undefined,
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@ -8,6 +8,7 @@ import {
|
||||
resolveApiKeyForProfile,
|
||||
resolveAuthProfileOrder,
|
||||
} from "../agents/auth-profiles.js";
|
||||
import { isNonSecretApiKeyMarker } from "../agents/model-auth-markers.js";
|
||||
import { getCustomProviderApiKey } from "../agents/model-auth.js";
|
||||
import { normalizeProviderId } from "../agents/model-selection.js";
|
||||
import { loadConfig } from "../config/config.js";
|
||||
@ -103,7 +104,7 @@ function resolveProviderApiKeyFromConfigAndStore(params: {
|
||||
|
||||
const cfg = loadConfig();
|
||||
const key = getCustomProviderApiKey(cfg, params.providerId);
|
||||
if (key) {
|
||||
if (key && !isNonSecretApiKeyMarker(key)) {
|
||||
return key;
|
||||
}
|
||||
|
||||
@ -122,9 +123,17 @@ function resolveProviderApiKeyFromConfigAndStore(params: {
|
||||
return undefined;
|
||||
}
|
||||
if (cred.type === "api_key") {
|
||||
return normalizeSecretInput(cred.key);
|
||||
const key = normalizeSecretInput(cred.key);
|
||||
if (key && !isNonSecretApiKeyMarker(key)) {
|
||||
return key;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
return normalizeSecretInput(cred.token);
|
||||
const token = normalizeSecretInput(cred.token);
|
||||
if (token && !isNonSecretApiKeyMarker(token)) {
|
||||
return token;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function resolveOAuthToken(params: {
|
||||
|
||||
@ -10,6 +10,7 @@ type AuditFixture = {
|
||||
configPath: string;
|
||||
authStorePath: string;
|
||||
authJsonPath: string;
|
||||
modelsPath: string;
|
||||
envPath: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
};
|
||||
@ -27,9 +28,11 @@ function resolveRuntimePathEnv(): string {
|
||||
|
||||
function hasFinding(
|
||||
report: Awaited<ReturnType<typeof runSecretsAudit>>,
|
||||
predicate: (entry: { code: string; file: string }) => boolean,
|
||||
predicate: (entry: { code: string; file: string; jsonPath?: string }) => boolean,
|
||||
): boolean {
|
||||
return report.findings.some((entry) => predicate(entry as { code: string; file: string }));
|
||||
return report.findings.some((entry) =>
|
||||
predicate(entry as { code: string; file: string; jsonPath?: string }),
|
||||
);
|
||||
}
|
||||
|
||||
async function createAuditFixture(): Promise<AuditFixture> {
|
||||
@ -38,6 +41,7 @@ async function createAuditFixture(): Promise<AuditFixture> {
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
const authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json");
|
||||
const authJsonPath = path.join(stateDir, "agents", "main", "agent", "auth.json");
|
||||
const modelsPath = path.join(stateDir, "agents", "main", "agent", "models.json");
|
||||
const envPath = path.join(stateDir, ".env");
|
||||
|
||||
await fs.mkdir(path.dirname(configPath), { recursive: true });
|
||||
@ -49,6 +53,7 @@ async function createAuditFixture(): Promise<AuditFixture> {
|
||||
configPath,
|
||||
authStorePath,
|
||||
authJsonPath,
|
||||
modelsPath,
|
||||
envPath,
|
||||
env: {
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
@ -85,6 +90,16 @@ async function seedAuditFixture(fixture: AuditFixture): Promise<void> {
|
||||
version: 1,
|
||||
profiles: Object.fromEntries(seededProfiles),
|
||||
});
|
||||
await writeJsonFile(fixture.modelsPath, {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "OPENAI_API_KEY",
|
||||
models: [{ id: "gpt-5", name: "gpt-5" }],
|
||||
},
|
||||
},
|
||||
});
|
||||
await fs.writeFile(fixture.envPath, "OPENAI_API_KEY=sk-openai-plaintext\n", "utf8");
|
||||
}
|
||||
|
||||
@ -254,4 +269,64 @@ describe("secrets audit", () => {
|
||||
const callCount = callLog.split("\n").filter((line) => line.trim().length > 0).length;
|
||||
expect(callCount).toBe(1);
|
||||
});
|
||||
|
||||
it("scans agent models.json files for plaintext provider apiKey values", async () => {
|
||||
await writeJsonFile(fixture.modelsPath, {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "sk-models-plaintext",
|
||||
models: [{ id: "gpt-5", name: "gpt-5" }],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const report = await runSecretsAudit({ env: fixture.env });
|
||||
expect(
|
||||
hasFinding(
|
||||
report,
|
||||
(entry) =>
|
||||
entry.code === "PLAINTEXT_FOUND" &&
|
||||
entry.file === fixture.modelsPath &&
|
||||
entry.jsonPath === "providers.openai.apiKey",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(report.filesScanned).toContain(fixture.modelsPath);
|
||||
});
|
||||
|
||||
it("does not flag models.json marker values as plaintext", async () => {
|
||||
await writeJsonFile(fixture.modelsPath, {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "OPENAI_API_KEY",
|
||||
models: [{ id: "gpt-5", name: "gpt-5" }],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const report = await runSecretsAudit({ env: fixture.env });
|
||||
expect(
|
||||
hasFinding(
|
||||
report,
|
||||
(entry) =>
|
||||
entry.code === "PLAINTEXT_FOUND" &&
|
||||
entry.file === fixture.modelsPath &&
|
||||
entry.jsonPath === "providers.openai.apiKey",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("reports malformed models.json as unresolved findings", async () => {
|
||||
await fs.writeFile(fixture.modelsPath, "{bad-json", "utf8");
|
||||
const report = await runSecretsAudit({ env: fixture.env });
|
||||
expect(
|
||||
hasFinding(
|
||||
report,
|
||||
(entry) => entry.code === "REF_UNRESOLVED" && entry.file === fixture.modelsPath,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { isNonSecretApiKeyMarker } from "../agents/model-auth-markers.js";
|
||||
import { normalizeProviderId } from "../agents/model-selection.js";
|
||||
import { resolveStateDir, type OpenClawConfig } from "../config/config.js";
|
||||
import { coerceSecretRef } from "../config/types.secrets.js";
|
||||
import { resolveSecretInputRef, type SecretRef } from "../config/types.secrets.js";
|
||||
import { resolveConfigDir, resolveUserPath } from "../utils.js";
|
||||
import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js";
|
||||
@ -23,6 +25,7 @@ import {
|
||||
import { isNonEmptyString, isRecord } from "./shared.js";
|
||||
import { describeUnknownError } from "./shared.js";
|
||||
import {
|
||||
listAgentModelsJsonPaths,
|
||||
listAuthProfileStorePaths,
|
||||
listLegacyAuthJsonPaths,
|
||||
parseEnvAssignmentValue,
|
||||
@ -315,6 +318,62 @@ function collectAuthJsonResidue(params: { stateDir: string; collector: AuditColl
|
||||
}
|
||||
}
|
||||
|
||||
function collectModelsJsonSecrets(params: {
|
||||
modelsJsonPath: string;
|
||||
collector: AuditCollector;
|
||||
}): void {
|
||||
if (!fs.existsSync(params.modelsJsonPath)) {
|
||||
return;
|
||||
}
|
||||
params.collector.filesScanned.add(params.modelsJsonPath);
|
||||
const parsedResult = readJsonObjectIfExists(params.modelsJsonPath);
|
||||
if (parsedResult.error) {
|
||||
addFinding(params.collector, {
|
||||
code: "REF_UNRESOLVED",
|
||||
severity: "error",
|
||||
file: params.modelsJsonPath,
|
||||
jsonPath: "<root>",
|
||||
message: `Invalid JSON in models.json: ${parsedResult.error}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const parsed = parsedResult.value;
|
||||
if (!parsed || !isRecord(parsed.providers)) {
|
||||
return;
|
||||
}
|
||||
for (const [providerId, providerValue] of Object.entries(parsed.providers)) {
|
||||
if (!isRecord(providerValue)) {
|
||||
continue;
|
||||
}
|
||||
const apiKey = providerValue.apiKey;
|
||||
if (coerceSecretRef(apiKey)) {
|
||||
addFinding(params.collector, {
|
||||
code: "REF_UNRESOLVED",
|
||||
severity: "error",
|
||||
file: params.modelsJsonPath,
|
||||
jsonPath: `providers.${providerId}.apiKey`,
|
||||
message: "models.json contains an unresolved SecretRef object; regenerate models.json.",
|
||||
provider: providerId,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!isNonEmptyString(apiKey)) {
|
||||
continue;
|
||||
}
|
||||
if (isNonSecretApiKeyMarker(apiKey)) {
|
||||
continue;
|
||||
}
|
||||
addFinding(params.collector, {
|
||||
code: "PLAINTEXT_FOUND",
|
||||
severity: "warn",
|
||||
file: params.modelsJsonPath,
|
||||
jsonPath: `providers.${providerId}.apiKey`,
|
||||
message: "models.json provider apiKey is stored as plaintext.",
|
||||
provider: providerId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function collectUnresolvedRefFindings(params: {
|
||||
collector: AuditCollector;
|
||||
config: OpenClawConfig;
|
||||
@ -497,6 +556,12 @@ export async function runSecretsAudit(
|
||||
defaults,
|
||||
});
|
||||
}
|
||||
for (const modelsJsonPath of listAgentModelsJsonPaths(config, stateDir)) {
|
||||
collectModelsJsonSecrets({
|
||||
modelsJsonPath,
|
||||
collector,
|
||||
});
|
||||
}
|
||||
await collectUnresolvedRefFindings({
|
||||
collector,
|
||||
config,
|
||||
|
||||
@ -31,6 +31,32 @@ export function listLegacyAuthJsonPaths(stateDir: string): string[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
export function listAgentModelsJsonPaths(config: OpenClawConfig, stateDir: string): string[] {
|
||||
const paths = new Set<string>();
|
||||
paths.add(path.join(resolveUserPath(stateDir), "agents", "main", "agent", "models.json"));
|
||||
|
||||
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
|
||||
if (fs.existsSync(agentsRoot)) {
|
||||
for (const entry of fs.readdirSync(agentsRoot, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
paths.add(path.join(agentsRoot, entry.name, "agent", "models.json"));
|
||||
}
|
||||
}
|
||||
|
||||
for (const agentId of listAgentIds(config)) {
|
||||
if (agentId === "main") {
|
||||
paths.add(path.join(resolveUserPath(stateDir), "agents", "main", "agent", "models.json"));
|
||||
continue;
|
||||
}
|
||||
const agentDir = resolveAgentDir(config, agentId);
|
||||
paths.add(path.join(resolveUserPath(agentDir), "models.json"));
|
||||
}
|
||||
|
||||
return [...paths];
|
||||
}
|
||||
|
||||
export function readJsonObjectIfExists(filePath: string): {
|
||||
value: Record<string, unknown> | null;
|
||||
error?: string;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user