import { Router, type IRouter, type Request, type Response } from "express"; import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; import type { MessageCreateParamsNonStreaming } from "@anthropic-ai/sdk/resources/messages"; import { config } from "../lib/config"; import { MODEL_INDEX, MODELS } from "../lib/models"; const router: IRouter = Router(); function openAIClient(): OpenAI | null { const provider = config.providers.openai; if (!provider.apiKey || !provider.baseURL) return null; return new OpenAI({ baseURL: provider.baseURL, apiKey: provider.apiKey, }); } function anthropicClient(): Anthropic | null { const provider = config.providers.anthropic; if (!provider.apiKey || !provider.baseURL) return null; return new Anthropic({ baseURL: provider.baseURL, apiKey: provider.apiKey, }); } function flush(res: Response): void { (res as Response & { flush?: () => void }).flush?.(); } function authorize(req: Request, res: Response): boolean { const expected = config.proxy.apiKey; const header = req.header("authorization") ?? ""; const match = header.match(/^Bearer\s+(.+)$/i); if (!match || match[1] !== expected) { res.status(401).json({ error: { message: "Invalid API key", type: "invalid_request_error" } }); return false; } return true; } router.get("/models", (req, res) => { if (!authorize(req, res)) return; const created = Math.floor(Date.now() / 1000); res.json({ object: "list", data: MODELS.map((m) => ({ id: m.id, object: "model", created, owned_by: m.provider, })), }); }); interface ChatMessage { role: "system" | "user" | "assistant" | "developer" | "tool"; content: unknown; name?: string; } interface ChatCompletionRequest { model: string; messages: ChatMessage[]; stream?: boolean; temperature?: number; top_p?: number; max_tokens?: number; max_completion_tokens?: number; stop?: string | string[]; frequency_penalty?: number; presence_penalty?: number; user?: string; reasoning_effort?: "minimal" | "low" | "medium" | "high"; thinking?: { type?: string; budget_tokens?: number } | null; [key: string]: unknown; } function effortToBudget(effort: string | undefined): number | null { switch (effort) { case "minimal": return 1024; case "low": return 2048; case "medium": return 8000; case "high": return 16000; default: return null; } } function isOpenAIModel(model: string): boolean { return /^(gpt|o\d|chatgpt)/i.test(model); } function isAnthropicModel(model: string): boolean { return /^claude/i.test(model); } function newId(): string { return "chatcmpl-" + Math.random().toString(36).slice(2, 14); } router.post("/chat/completions", async (req, res) => { if (!authorize(req, res)) return; const body = req.body as ChatCompletionRequest; if (!body || typeof body !== "object" || !body.model || !Array.isArray(body.messages)) { res.status(400).json({ error: { message: "model and messages are required", type: "invalid_request_error" }, }); return; } const requestedId = body.model; const entry = MODEL_INDEX.get(requestedId); let provider: "openai" | "anthropic"; if (entry) { body.model = entry.baseModel; if (entry.effort && !body.reasoning_effort) { body.reasoning_effort = entry.effort; } provider = entry.provider; } else if (isOpenAIModel(requestedId)) { provider = "openai"; } else if (isAnthropicModel(requestedId)) { provider = "anthropic"; } else { res.status(400).json({ error: { message: `Unsupported model: ${requestedId}`, type: "invalid_request_error", }, }); return; } const stream = body.stream === true; if (provider === "openai") { await handleOpenAI(req, res, body, stream); } else { await handleAnthropic(req, res, body, stream); } }); function setupSse(res: Response): NodeJS.Timeout { res.setHeader("Content-Type", "text/event-stream"); res.setHeader("Cache-Control", "no-cache, no-transform"); res.setHeader("Connection", "keep-alive"); res.setHeader("X-Accel-Buffering", "no"); res.flushHeaders(); const keepalive = setInterval(() => { try { res.write(": keepalive\n\n"); flush(res); } catch { // ignore } }, 5000); return keepalive; } async function handleOpenAI( req: Request, res: Response, body: ChatCompletionRequest, stream: boolean, ): Promise { const client = openAIClient(); if (!client) { providerNotConfigured(res, "openai"); return; } const { stream: _ignored, ...rest } = body; if (!stream) { try { const completion = await client.chat.completions.create({ ...rest, stream: false, } as Parameters[0]); res.json(completion); } catch (err) { handleProviderError(req, res, err, "openai"); } return; } let keepalive: NodeJS.Timeout | undefined; let clientGone = false; res.on("close", () => { if (!res.writableEnded) clientGone = true; if (keepalive) clearInterval(keepalive); }); try { const upstream = await client.chat.completions.create({ ...rest, stream: true, } as Parameters[0]); keepalive = setupSse(res); for await (const chunk of upstream as AsyncIterable) { if (clientGone) break; res.write(`data: ${JSON.stringify(chunk)}\n\n`); flush(res); } if (!clientGone) { res.write("data: [DONE]\n\n"); flush(res); } } catch (err) { if (!res.headersSent) { handleProviderError(req, res, err, "openai"); return; } req.log?.error({ err }, "openai stream error"); try { res.write( `data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`, ); flush(res); } catch { // ignore } } finally { if (keepalive) clearInterval(keepalive); if (!clientGone) { try { res.end(); } catch { // ignore } } } } interface AnthropicConvertedMessages { system: string | undefined; messages: Array<{ role: "user" | "assistant"; content: unknown }>; } function convertMessagesForAnthropic(messages: ChatMessage[]): AnthropicConvertedMessages { const systemParts: string[] = []; const out: Array<{ role: "user" | "assistant"; content: unknown }> = []; for (const msg of messages) { if (msg.role === "system" || msg.role === "developer") { if (typeof msg.content === "string") { systemParts.push(msg.content); } else if (Array.isArray(msg.content)) { for (const part of msg.content as Array<{ type?: string; text?: string }>) { if (part?.type === "text" && typeof part.text === "string") { systemParts.push(part.text); } } } } else if (msg.role === "user" || msg.role === "assistant") { out.push({ role: msg.role, content: msg.content }); } // ignore tool messages for this passthrough } return { system: systemParts.length > 0 ? systemParts.join("\n\n") : undefined, messages: out, }; } async function handleAnthropic( req: Request, res: Response, body: ChatCompletionRequest, stream: boolean, ): Promise { const client = anthropicClient(); if (!client) { providerNotConfigured(res, "anthropic"); return; } const { system, messages } = convertMessagesForAnthropic(body.messages); let maxTokens = body.max_completion_tokens ?? body.max_tokens ?? 8192; let thinkingParam: { type: "enabled"; budget_tokens: number } | null = null; if (body.thinking && (body.thinking.type === "enabled" || typeof body.thinking.budget_tokens === "number")) { const budget = Math.max(1024, body.thinking.budget_tokens ?? 4096); thinkingParam = { type: "enabled", budget_tokens: budget }; } else { const budget = effortToBudget(body.reasoning_effort); if (budget !== null) { thinkingParam = { type: "enabled", budget_tokens: budget }; } } if (thinkingParam) { if (maxTokens <= thinkingParam.budget_tokens) { maxTokens = thinkingParam.budget_tokens + 4096; } } const baseParams: Record = { model: body.model, max_tokens: maxTokens, messages, }; if (system) baseParams["system"] = system; if (thinkingParam) { baseParams["thinking"] = thinkingParam; // Anthropic requires temperature=1 (or unset) and rejects top_p when thinking is enabled. } else { if (typeof body.temperature === "number") baseParams["temperature"] = body.temperature; if (typeof body.top_p === "number") baseParams["top_p"] = body.top_p; } if (body.stop !== undefined) { baseParams["stop_sequences"] = Array.isArray(body.stop) ? body.stop : [body.stop]; } if (!stream) { try { const message = await client.messages.create( baseParams as unknown as MessageCreateParamsNonStreaming, ); let text = ""; let reasoning = ""; for (const b of message.content ?? []) { if (b.type === "text") text += b.text; else if (b.type === "thinking") reasoning += (b as { thinking?: string }).thinking ?? ""; } const finishReason = mapAnthropicStopReason(message.stop_reason ?? null); const assistantMessage: Record = { role: "assistant", content: text }; if (reasoning) assistantMessage["reasoning_content"] = reasoning; res.json({ id: message.id, object: "chat.completion", created: Math.floor(Date.now() / 1000), model: message.model, choices: [ { index: 0, message: assistantMessage, finish_reason: finishReason, }, ], usage: { prompt_tokens: message.usage?.input_tokens ?? 0, completion_tokens: message.usage?.output_tokens ?? 0, total_tokens: (message.usage?.input_tokens ?? 0) + (message.usage?.output_tokens ?? 0), }, }); } catch (err) { handleProviderError(req, res, err, "anthropic"); } return; } let keepalive: NodeJS.Timeout | undefined; let clientGone = false; res.on("close", () => { if (!res.writableEnded) clientGone = true; if (keepalive) clearInterval(keepalive); }); const id = newId(); const created = Math.floor(Date.now() / 1000); try { const upstream = client.messages.stream( baseParams as Parameters[0], ); keepalive = setupSse(res); res.write( `data: ${JSON.stringify({ id, object: "chat.completion.chunk", created, model: body.model, choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], })}\n\n`, ); flush(res); let stopReason: string | null = null; for await (const event of upstream) { if (clientGone) break; if (event.type === "content_block_delta") { let delta: Record | null = null; if (event.delta.type === "text_delta") { delta = { content: event.delta.text }; } else if (event.delta.type === "thinking_delta") { const td = event.delta as unknown as { thinking?: string }; if (td.thinking) delta = { reasoning_content: td.thinking }; } if (delta) { const chunk = { id, object: "chat.completion.chunk", created, model: body.model, choices: [{ index: 0, delta, finish_reason: null }], }; res.write(`data: ${JSON.stringify(chunk)}\n\n`); flush(res); } } else if (event.type === "message_delta") { const md = event as unknown as { delta?: { stop_reason?: string | null } }; if (md.delta?.stop_reason) { stopReason = md.delta.stop_reason; } } } if (!clientGone) { const finishChunk = { id, object: "chat.completion.chunk", created, model: body.model, choices: [ { index: 0, delta: {}, finish_reason: mapAnthropicStopReason(stopReason), }, ], }; res.write(`data: ${JSON.stringify(finishChunk)}\n\n`); res.write("data: [DONE]\n\n"); flush(res); } } catch (err) { if (!res.headersSent) { handleProviderError(req, res, err, "anthropic"); return; } req.log?.error({ err }, "anthropic stream error"); try { res.write( `data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`, ); flush(res); } catch { // ignore } } finally { if (keepalive) clearInterval(keepalive); if (!clientGone) { try { res.end(); } catch { // ignore } } } } function mapAnthropicStopReason(reason: string | null): string { switch (reason) { case "end_turn": case "stop_sequence": return "stop"; case "max_tokens": return "length"; case "tool_use": return "tool_calls"; default: return "stop"; } } function errorMessage(err: unknown): string { if (err instanceof Error) return err.message; if (typeof err === "string") return err; try { return JSON.stringify(err); } catch { return "unknown error"; } } function providerNotConfigured(res: Response, provider: "openai" | "anthropic"): void { res.status(503).json({ error: { message: `${provider} provider is not configured. Pass the matching Replit AI Integrations CLI flags, not public provider endpoints.`, type: "server_error", }, }); } function handleProviderError( req: Request, res: Response, err: unknown, provider: string, ): void { req.log?.error({ err, provider }, "provider error"); const status = typeof (err as { status?: number })?.status === "number" ? (err as { status: number }).status : 500; if (!res.headersSent) { res.status(status).json({ error: { message: errorMessage(err), type: "upstream_error", provider, }, }); } } export default router;