From 6710a2be61f8ef8cf49bf960f2ad73b96c9c28fc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 17 Mar 2026 21:35:13 -0700 Subject: [PATCH] Image generation: add fal provider (#49454) --- .github/labeler.yml | 4 + CHANGELOG.md | 1 + extensions/fal/index.ts | 44 +++++ extensions/fal/onboard.ts | 21 +++ extensions/fal/openclaw.plugin.json | 27 +++ extensions/fal/package.json | 12 ++ pnpm-lock.yaml | 2 + src/image-generation/providers/fal.test.ts | 151 ++++++++++++++++ src/image-generation/providers/fal.ts | 198 +++++++++++++++++++++ src/plugin-sdk/image-generation.ts | 1 + 10 files changed, 461 insertions(+) create mode 100644 extensions/fal/index.ts create mode 100644 extensions/fal/onboard.ts create mode 100644 extensions/fal/openclaw.plugin.json create mode 100644 extensions/fal/package.json create mode 100644 src/image-generation/providers/fal.test.ts create mode 100644 src/image-generation/providers/fal.ts diff --git a/.github/labeler.yml b/.github/labeler.yml index b6422060fea..7dcc038de4c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -314,3 +314,7 @@ - changed-files: - any-glob-to-any-file: - "extensions/xiaomi/**" +"extensions: fal": + - changed-files: + - any-glob-to-any-file: + - "extensions/fal/**" diff --git a/CHANGELOG.md b/CHANGELOG.md index 31378788a67..bd6a4c7a34e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,6 +136,7 @@ Docs: https://docs.openclaw.ai - Plugins/runtime-api: pin extension runtime-api export seams with explicit guardrail coverage so future surface creep becomes a deliberate diff. Thanks @vincentkoc. - Telegram/security: add regression coverage proving pinned fallback host overrides stay bound to Telegram and delegate non-matching hostnames back to the original lookup path. Thanks @vincentkoc. - Secrets/exec refs: require explicit `--allow-exec` for `secrets apply` write plans that contain exec SecretRefs/providers, and align audit/configure/apply dry-run behavior to skip exec checks unless opted in to prevent unexpected command side effects. (#49417) Thanks @restriction and @joshavant. +- Tools/image generation: add bundled fal image generation support so `image_generate` can target `fal/*` models with `FAL_KEY`, including single-image edit flows via FLUX image-to-image. Thanks @vincentkoc. ### Breaking diff --git a/extensions/fal/index.ts b/extensions/fal/index.ts new file mode 100644 index 00000000000..e1eaf0a9c36 --- /dev/null +++ b/extensions/fal/index.ts @@ -0,0 +1,44 @@ +import { definePluginEntry } from "openclaw/plugin-sdk/core"; +import { buildFalImageGenerationProvider } from "openclaw/plugin-sdk/image-generation"; +import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth"; +import { applyFalConfig, FAL_DEFAULT_IMAGE_MODEL_REF } from "./onboard.js"; + +const PROVIDER_ID = "fal"; + +export default definePluginEntry({ + id: PROVIDER_ID, + name: "fal Provider", + description: "Bundled fal image generation provider", + register(api) { + api.registerProvider({ + id: PROVIDER_ID, + label: "fal", + docsPath: "/providers/models", + envVars: ["FAL_KEY"], + auth: [ + createProviderApiKeyAuthMethod({ + providerId: PROVIDER_ID, + methodId: "api-key", + label: "fal API key", + hint: "Image generation API key", + optionKey: "falApiKey", + flagName: "--fal-api-key", + envVar: "FAL_KEY", + promptMessage: "Enter fal API key", + defaultModel: FAL_DEFAULT_IMAGE_MODEL_REF, + expectedProviders: ["fal"], + applyConfig: (cfg) => applyFalConfig(cfg), + wizard: { + choiceId: "fal-api-key", + choiceLabel: "fal API key", + choiceHint: "Image generation API key", + groupId: "fal", + groupLabel: "fal", + groupHint: "Image generation", + }, + }), + ], + }); + api.registerImageGenerationProvider(buildFalImageGenerationProvider()); + }, +}); diff --git a/extensions/fal/onboard.ts b/extensions/fal/onboard.ts new file mode 100644 index 00000000000..3478599ae59 --- /dev/null +++ b/extensions/fal/onboard.ts @@ -0,0 +1,21 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard"; + +export const FAL_DEFAULT_IMAGE_MODEL_REF = "fal/fal-ai/flux/dev"; + +export function applyFalConfig(cfg: OpenClawConfig): OpenClawConfig { + if (cfg.agents?.defaults?.imageGenerationModel) { + return cfg; + } + return { + ...cfg, + agents: { + ...cfg.agents, + defaults: { + ...cfg.agents?.defaults, + imageGenerationModel: { + primary: FAL_DEFAULT_IMAGE_MODEL_REF, + }, + }, + }, + }; +} diff --git a/extensions/fal/openclaw.plugin.json b/extensions/fal/openclaw.plugin.json new file mode 100644 index 00000000000..52128c23fac --- /dev/null +++ b/extensions/fal/openclaw.plugin.json @@ -0,0 +1,27 @@ +{ + "id": "fal", + "providers": ["fal"], + "providerAuthEnvVars": { + "fal": ["FAL_KEY"] + }, + "providerAuthChoices": [ + { + "provider": "fal", + "method": "api-key", + "choiceId": "fal-api-key", + "choiceLabel": "fal API key", + "groupId": "fal", + "groupLabel": "fal", + "groupHint": "Image generation", + "optionKey": "falApiKey", + "cliFlag": "--fal-api-key", + "cliOption": "--fal-api-key ", + "cliDescription": "fal API key" + } + ], + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/fal/package.json b/extensions/fal/package.json new file mode 100644 index 00000000000..ebe51568a10 --- /dev/null +++ b/extensions/fal/package.json @@ -0,0 +1,12 @@ +{ + "name": "@openclaw/fal-provider", + "version": "2026.3.14", + "private": true, + "description": "OpenClaw fal provider plugin", + "type": "module", + "openclaw": { + "extensions": [ + "./index.ts" + ] + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4bb99278823..b43381e461c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -346,6 +346,8 @@ importers: extensions/elevenlabs: {} + extensions/fal: {} + extensions/feishu: dependencies: '@larksuiteoapi/node-sdk': diff --git a/src/image-generation/providers/fal.test.ts b/src/image-generation/providers/fal.test.ts new file mode 100644 index 00000000000..c610c1b9c0c --- /dev/null +++ b/src/image-generation/providers/fal.test.ts @@ -0,0 +1,151 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as modelAuth from "../../agents/model-auth.js"; +import { buildFalImageGenerationProvider } from "./fal.js"; + +describe("fal image-generation provider", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("generates image buffers from the fal sync API", async () => { + vi.spyOn(modelAuth, "resolveApiKeyForProvider").mockResolvedValue({ + apiKey: "fal-test-key", + source: "env", + mode: "api-key", + }); + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + images: [ + { + url: "https://v3.fal.media/files/example/generated.png", + content_type: "image/png", + }, + ], + prompt: "draw a cat", + }), + }) + .mockResolvedValueOnce({ + ok: true, + headers: new Headers({ "content-type": "image/png" }), + arrayBuffer: async () => Buffer.from("png-data"), + }); + vi.stubGlobal("fetch", fetchMock); + + const provider = buildFalImageGenerationProvider(); + const result = await provider.generateImage({ + provider: "fal", + model: "fal-ai/flux/dev", + prompt: "draw a cat", + cfg: {}, + count: 2, + size: "1536x1024", + }); + + expect(fetchMock).toHaveBeenNthCalledWith( + 1, + "https://fal.run/fal-ai/flux/dev", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + prompt: "draw a cat", + image_size: { width: 1536, height: 1024 }, + num_images: 2, + output_format: "png", + }), + }), + ); + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + "https://v3.fal.media/files/example/generated.png", + ); + expect(result).toEqual({ + images: [ + { + buffer: Buffer.from("png-data"), + mimeType: "image/png", + fileName: "image-1.png", + }, + ], + model: "fal-ai/flux/dev", + metadata: { prompt: "draw a cat" }, + }); + }); + + it("uses image-to-image endpoint and data-uri input for edits", async () => { + vi.spyOn(modelAuth, "resolveApiKeyForProvider").mockResolvedValue({ + apiKey: "fal-test-key", + source: "env", + mode: "api-key", + }); + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + images: [{ url: "https://v3.fal.media/files/example/edited.png" }], + }), + }) + .mockResolvedValueOnce({ + ok: true, + headers: new Headers({ "content-type": "image/png" }), + arrayBuffer: async () => Buffer.from("edited-data"), + }); + vi.stubGlobal("fetch", fetchMock); + + const provider = buildFalImageGenerationProvider(); + await provider.generateImage({ + provider: "fal", + model: "fal-ai/flux/dev", + prompt: "turn this into a noir poster", + cfg: {}, + resolution: "2K", + inputImages: [ + { + buffer: Buffer.from("source-image"), + mimeType: "image/jpeg", + fileName: "source.jpg", + }, + ], + }); + + expect(fetchMock).toHaveBeenNthCalledWith( + 1, + "https://fal.run/fal-ai/flux/dev/image-to-image", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + prompt: "turn this into a noir poster", + image_size: { width: 2048, height: 2048 }, + num_images: 1, + output_format: "png", + image_url: `data:image/jpeg;base64,${Buffer.from("source-image").toString("base64")}`, + }), + }), + ); + }); + + it("rejects multi-image edit requests for now", async () => { + vi.spyOn(modelAuth, "resolveApiKeyForProvider").mockResolvedValue({ + apiKey: "fal-test-key", + source: "env", + mode: "api-key", + }); + + const provider = buildFalImageGenerationProvider(); + await expect( + provider.generateImage({ + provider: "fal", + model: "fal-ai/flux/dev", + prompt: "combine these", + cfg: {}, + inputImages: [ + { buffer: Buffer.from("one"), mimeType: "image/png" }, + { buffer: Buffer.from("two"), mimeType: "image/png" }, + ], + }), + ).rejects.toThrow("at most one reference image"); + }); +}); diff --git a/src/image-generation/providers/fal.ts b/src/image-generation/providers/fal.ts new file mode 100644 index 00000000000..b9bd5517651 --- /dev/null +++ b/src/image-generation/providers/fal.ts @@ -0,0 +1,198 @@ +import { resolveApiKeyForProvider } from "../../agents/model-auth.js"; +import type { ImageGenerationProviderPlugin } from "../../plugins/types.js"; +import type { GeneratedImageAsset } from "../types.js"; + +const DEFAULT_FAL_BASE_URL = "https://fal.run"; +const DEFAULT_FAL_IMAGE_MODEL = "fal-ai/flux/dev"; +const DEFAULT_FAL_EDIT_SUBPATH = "image-to-image"; +const DEFAULT_OUTPUT_SIZE = "square_hd"; +const DEFAULT_OUTPUT_FORMAT = "png"; + +type FalGeneratedImage = { + url?: string; + content_type?: string; +}; + +type FalImageGenerationResponse = { + images?: FalGeneratedImage[]; + prompt?: string; +}; + +type FalImageSize = string | { width: number; height: number }; + +function resolveFalBaseUrl(cfg: Parameters[0]["cfg"]): string { + const direct = cfg?.models?.providers?.fal?.baseUrl?.trim(); + return (direct || DEFAULT_FAL_BASE_URL).replace(/\/+$/u, ""); +} + +function ensureFalModelPath(model: string | undefined, hasInputImages: boolean): string { + const trimmed = model?.trim() || DEFAULT_FAL_IMAGE_MODEL; + if (!hasInputImages) { + return trimmed; + } + if ( + trimmed.endsWith(`/${DEFAULT_FAL_EDIT_SUBPATH}`) || + trimmed.endsWith("/edit") || + trimmed.includes("/image-to-image/") + ) { + return trimmed; + } + return `${trimmed}/${DEFAULT_FAL_EDIT_SUBPATH}`; +} + +function parseSize(raw: string | undefined): { width: number; height: number } | null { + const trimmed = raw?.trim(); + if (!trimmed) { + return null; + } + const match = /^(\d{2,5})x(\d{2,5})$/iu.exec(trimmed); + if (!match) { + return null; + } + const width = Number.parseInt(match[1] ?? "", 10); + const height = Number.parseInt(match[2] ?? "", 10); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return null; + } + return { width, height }; +} + +function mapResolutionToSize(resolution: "1K" | "2K" | "4K" | undefined): FalImageSize | undefined { + if (!resolution) { + return undefined; + } + const edge = resolution === "4K" ? 4096 : resolution === "2K" ? 2048 : 1024; + return { width: edge, height: edge }; +} + +function resolveFalImageSize(params: { + size?: string; + resolution?: "1K" | "2K" | "4K"; +}): FalImageSize { + const parsed = parseSize(params.size); + if (parsed) { + return parsed; + } + return mapResolutionToSize(params.resolution) ?? DEFAULT_OUTPUT_SIZE; +} + +function toDataUri(buffer: Buffer, mimeType: string): string { + return `data:${mimeType};base64,${buffer.toString("base64")}`; +} + +function fileExtensionForMimeType(mimeType: string | undefined): string { + const normalized = mimeType?.toLowerCase().trim(); + if (!normalized) { + return "png"; + } + if (normalized.includes("jpeg")) { + return "jpg"; + } + const slashIndex = normalized.indexOf("/"); + return slashIndex >= 0 ? normalized.slice(slashIndex + 1) || "png" : "png"; +} + +async function fetchImageBuffer(url: string): Promise<{ buffer: Buffer; mimeType: string }> { + const response = await fetch(url); + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error( + `fal image download failed (${response.status}): ${text || response.statusText}`, + ); + } + const mimeType = response.headers.get("content-type")?.trim() || "image/png"; + const arrayBuffer = await response.arrayBuffer(); + return { buffer: Buffer.from(arrayBuffer), mimeType }; +} + +export function buildFalImageGenerationProvider(): ImageGenerationProviderPlugin { + return { + id: "fal", + label: "fal", + defaultModel: DEFAULT_FAL_IMAGE_MODEL, + models: [DEFAULT_FAL_IMAGE_MODEL, `${DEFAULT_FAL_IMAGE_MODEL}/${DEFAULT_FAL_EDIT_SUBPATH}`], + supportedSizes: ["1024x1024", "1024x1536", "1536x1024", "1024x1792", "1792x1024"], + supportedResolutions: ["1K", "2K", "4K"], + supportsImageEditing: true, + async generateImage(req) { + const auth = await resolveApiKeyForProvider({ + provider: "fal", + cfg: req.cfg, + agentDir: req.agentDir, + store: req.authStore, + }); + if (!auth.apiKey) { + throw new Error("fal API key missing"); + } + if ((req.inputImages?.length ?? 0) > 1) { + throw new Error("fal image generation currently supports at most one reference image"); + } + + const imageSize = resolveFalImageSize({ + size: req.size, + resolution: req.resolution, + }); + const hasInputImages = (req.inputImages?.length ?? 0) > 0; + const model = ensureFalModelPath(req.model, hasInputImages); + const requestBody: Record = { + prompt: req.prompt, + image_size: imageSize, + num_images: req.count ?? 1, + output_format: DEFAULT_OUTPUT_FORMAT, + }; + + if (hasInputImages) { + const [input] = req.inputImages ?? []; + if (!input) { + throw new Error("fal image edit request missing reference image"); + } + requestBody.image_url = toDataUri(input.buffer, input.mimeType); + } + + const response = await fetch(`${resolveFalBaseUrl(req.cfg)}/${model}`, { + method: "POST", + headers: { + Authorization: `Key ${auth.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error( + `fal image generation failed (${response.status}): ${text || response.statusText}`, + ); + } + + const payload = (await response.json()) as FalImageGenerationResponse; + const images: GeneratedImageAsset[] = []; + let imageIndex = 0; + for (const entry of payload.images ?? []) { + const url = entry.url?.trim(); + if (!url) { + continue; + } + const downloaded = await fetchImageBuffer(url); + imageIndex += 1; + images.push({ + buffer: downloaded.buffer, + mimeType: downloaded.mimeType, + fileName: `image-${imageIndex}.${fileExtensionForMimeType( + downloaded.mimeType || entry.content_type, + )}`, + }); + } + + if (images.length === 0) { + throw new Error("fal image generation response missing image data"); + } + + return { + images, + model, + metadata: payload.prompt ? { prompt: payload.prompt } : undefined, + }; + }, + }; +} diff --git a/src/plugin-sdk/image-generation.ts b/src/plugin-sdk/image-generation.ts index 25fde2e9d2b..c999bbbaabe 100644 --- a/src/plugin-sdk/image-generation.ts +++ b/src/plugin-sdk/image-generation.ts @@ -9,5 +9,6 @@ export type { ImageGenerationSourceImage, } from "../image-generation/types.js"; +export { buildFalImageGenerationProvider } from "../image-generation/providers/fal.js"; export { buildGoogleImageGenerationProvider } from "../image-generation/providers/google.js"; export { buildOpenAIImageGenerationProvider } from "../image-generation/providers/openai.js";