526 lines
14 KiB
TypeScript
526 lines
14 KiB
TypeScript
import { Router, type IRouter, type Request, type Response } from "express";
|
|
import OpenAI from "openai";
|
|
import Anthropic from "@anthropic-ai/sdk";
|
|
import type { MessageCreateParamsNonStreaming } from "@anthropic-ai/sdk/resources/messages";
|
|
import { config } from "../lib/config";
|
|
import { MODEL_INDEX, MODELS } from "../lib/models";
|
|
|
|
const router: IRouter = Router();
|
|
|
|
function openAIClient(): OpenAI | null {
|
|
const provider = config.providers.openai;
|
|
if (!provider.apiKey || !provider.baseURL) return null;
|
|
return new OpenAI({
|
|
baseURL: provider.baseURL,
|
|
apiKey: provider.apiKey,
|
|
});
|
|
}
|
|
|
|
function anthropicClient(): Anthropic | null {
|
|
const provider = config.providers.anthropic;
|
|
if (!provider.apiKey || !provider.baseURL) return null;
|
|
return new Anthropic({
|
|
baseURL: provider.baseURL,
|
|
apiKey: provider.apiKey,
|
|
});
|
|
}
|
|
|
|
function flush(res: Response): void {
|
|
(res as Response & { flush?: () => void }).flush?.();
|
|
}
|
|
|
|
function authorize(req: Request, res: Response): boolean {
|
|
const expected = config.proxy.apiKey;
|
|
const header = req.header("authorization") ?? "";
|
|
const match = header.match(/^Bearer\s+(.+)$/i);
|
|
if (!match || match[1] !== expected) {
|
|
res.status(401).json({ error: { message: "Invalid API key", type: "invalid_request_error" } });
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
router.get("/models", (req, res) => {
|
|
if (!authorize(req, res)) return;
|
|
const created = Math.floor(Date.now() / 1000);
|
|
res.json({
|
|
object: "list",
|
|
data: MODELS.map((m) => ({
|
|
id: m.id,
|
|
object: "model",
|
|
created,
|
|
owned_by: m.provider,
|
|
})),
|
|
});
|
|
});
|
|
|
|
interface ChatMessage {
|
|
role: "system" | "user" | "assistant" | "developer" | "tool";
|
|
content: unknown;
|
|
name?: string;
|
|
}
|
|
|
|
interface ChatCompletionRequest {
|
|
model: string;
|
|
messages: ChatMessage[];
|
|
stream?: boolean;
|
|
temperature?: number;
|
|
top_p?: number;
|
|
max_tokens?: number;
|
|
max_completion_tokens?: number;
|
|
stop?: string | string[];
|
|
frequency_penalty?: number;
|
|
presence_penalty?: number;
|
|
user?: string;
|
|
reasoning_effort?: "minimal" | "low" | "medium" | "high";
|
|
thinking?: { type?: string; budget_tokens?: number } | null;
|
|
[key: string]: unknown;
|
|
}
|
|
|
|
function effortToBudget(effort: string | undefined): number | null {
|
|
switch (effort) {
|
|
case "minimal":
|
|
return 1024;
|
|
case "low":
|
|
return 2048;
|
|
case "medium":
|
|
return 8000;
|
|
case "high":
|
|
return 16000;
|
|
default:
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function isOpenAIModel(model: string): boolean {
|
|
return /^(gpt|o\d|chatgpt)/i.test(model);
|
|
}
|
|
|
|
function isAnthropicModel(model: string): boolean {
|
|
return /^claude/i.test(model);
|
|
}
|
|
|
|
function newId(): string {
|
|
return "chatcmpl-" + Math.random().toString(36).slice(2, 14);
|
|
}
|
|
|
|
router.post("/chat/completions", async (req, res) => {
|
|
if (!authorize(req, res)) return;
|
|
|
|
const body = req.body as ChatCompletionRequest;
|
|
if (!body || typeof body !== "object" || !body.model || !Array.isArray(body.messages)) {
|
|
res.status(400).json({
|
|
error: { message: "model and messages are required", type: "invalid_request_error" },
|
|
});
|
|
return;
|
|
}
|
|
|
|
const requestedId = body.model;
|
|
const entry = MODEL_INDEX.get(requestedId);
|
|
|
|
let provider: "openai" | "anthropic";
|
|
if (entry) {
|
|
body.model = entry.baseModel;
|
|
if (entry.effort && !body.reasoning_effort) {
|
|
body.reasoning_effort = entry.effort;
|
|
}
|
|
provider = entry.provider;
|
|
} else if (isOpenAIModel(requestedId)) {
|
|
provider = "openai";
|
|
} else if (isAnthropicModel(requestedId)) {
|
|
provider = "anthropic";
|
|
} else {
|
|
res.status(400).json({
|
|
error: {
|
|
message: `Unsupported model: ${requestedId}`,
|
|
type: "invalid_request_error",
|
|
},
|
|
});
|
|
return;
|
|
}
|
|
|
|
const stream = body.stream === true;
|
|
|
|
if (provider === "openai") {
|
|
await handleOpenAI(req, res, body, stream);
|
|
} else {
|
|
await handleAnthropic(req, res, body, stream);
|
|
}
|
|
});
|
|
|
|
function setupSse(res: Response): NodeJS.Timeout {
|
|
res.setHeader("Content-Type", "text/event-stream");
|
|
res.setHeader("Cache-Control", "no-cache, no-transform");
|
|
res.setHeader("Connection", "keep-alive");
|
|
res.setHeader("X-Accel-Buffering", "no");
|
|
res.flushHeaders();
|
|
const keepalive = setInterval(() => {
|
|
try {
|
|
res.write(": keepalive\n\n");
|
|
flush(res);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}, 5000);
|
|
return keepalive;
|
|
}
|
|
|
|
async function handleOpenAI(
|
|
req: Request,
|
|
res: Response,
|
|
body: ChatCompletionRequest,
|
|
stream: boolean,
|
|
): Promise<void> {
|
|
const client = openAIClient();
|
|
if (!client) {
|
|
providerNotConfigured(res, "openai");
|
|
return;
|
|
}
|
|
|
|
const { stream: _ignored, ...rest } = body;
|
|
|
|
if (!stream) {
|
|
try {
|
|
const completion = await client.chat.completions.create({
|
|
...rest,
|
|
stream: false,
|
|
} as Parameters<typeof client.chat.completions.create>[0]);
|
|
res.json(completion);
|
|
} catch (err) {
|
|
handleProviderError(req, res, err, "openai");
|
|
}
|
|
return;
|
|
}
|
|
|
|
let keepalive: NodeJS.Timeout | undefined;
|
|
let clientGone = false;
|
|
res.on("close", () => {
|
|
if (!res.writableEnded) clientGone = true;
|
|
if (keepalive) clearInterval(keepalive);
|
|
});
|
|
|
|
try {
|
|
const upstream = await client.chat.completions.create({
|
|
...rest,
|
|
stream: true,
|
|
} as Parameters<typeof client.chat.completions.create>[0]);
|
|
|
|
keepalive = setupSse(res);
|
|
|
|
for await (const chunk of upstream as AsyncIterable<unknown>) {
|
|
if (clientGone) break;
|
|
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
|
flush(res);
|
|
}
|
|
|
|
if (!clientGone) {
|
|
res.write("data: [DONE]\n\n");
|
|
flush(res);
|
|
}
|
|
} catch (err) {
|
|
if (!res.headersSent) {
|
|
handleProviderError(req, res, err, "openai");
|
|
return;
|
|
}
|
|
req.log?.error({ err }, "openai stream error");
|
|
try {
|
|
res.write(
|
|
`data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`,
|
|
);
|
|
flush(res);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
} finally {
|
|
if (keepalive) clearInterval(keepalive);
|
|
if (!clientGone) {
|
|
try {
|
|
res.end();
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
interface AnthropicConvertedMessages {
|
|
system: string | undefined;
|
|
messages: Array<{ role: "user" | "assistant"; content: unknown }>;
|
|
}
|
|
|
|
function convertMessagesForAnthropic(messages: ChatMessage[]): AnthropicConvertedMessages {
|
|
const systemParts: string[] = [];
|
|
const out: Array<{ role: "user" | "assistant"; content: unknown }> = [];
|
|
for (const msg of messages) {
|
|
if (msg.role === "system" || msg.role === "developer") {
|
|
if (typeof msg.content === "string") {
|
|
systemParts.push(msg.content);
|
|
} else if (Array.isArray(msg.content)) {
|
|
for (const part of msg.content as Array<{ type?: string; text?: string }>) {
|
|
if (part?.type === "text" && typeof part.text === "string") {
|
|
systemParts.push(part.text);
|
|
}
|
|
}
|
|
}
|
|
} else if (msg.role === "user" || msg.role === "assistant") {
|
|
out.push({ role: msg.role, content: msg.content });
|
|
}
|
|
// ignore tool messages for this passthrough
|
|
}
|
|
return {
|
|
system: systemParts.length > 0 ? systemParts.join("\n\n") : undefined,
|
|
messages: out,
|
|
};
|
|
}
|
|
|
|
async function handleAnthropic(
|
|
req: Request,
|
|
res: Response,
|
|
body: ChatCompletionRequest,
|
|
stream: boolean,
|
|
): Promise<void> {
|
|
const client = anthropicClient();
|
|
if (!client) {
|
|
providerNotConfigured(res, "anthropic");
|
|
return;
|
|
}
|
|
|
|
const { system, messages } = convertMessagesForAnthropic(body.messages);
|
|
let maxTokens =
|
|
body.max_completion_tokens ??
|
|
body.max_tokens ??
|
|
8192;
|
|
|
|
let thinkingParam: { type: "enabled"; budget_tokens: number } | null = null;
|
|
if (body.thinking && (body.thinking.type === "enabled" || typeof body.thinking.budget_tokens === "number")) {
|
|
const budget = Math.max(1024, body.thinking.budget_tokens ?? 4096);
|
|
thinkingParam = { type: "enabled", budget_tokens: budget };
|
|
} else {
|
|
const budget = effortToBudget(body.reasoning_effort);
|
|
if (budget !== null) {
|
|
thinkingParam = { type: "enabled", budget_tokens: budget };
|
|
}
|
|
}
|
|
|
|
if (thinkingParam) {
|
|
if (maxTokens <= thinkingParam.budget_tokens) {
|
|
maxTokens = thinkingParam.budget_tokens + 4096;
|
|
}
|
|
}
|
|
|
|
const baseParams: Record<string, unknown> = {
|
|
model: body.model,
|
|
max_tokens: maxTokens,
|
|
messages,
|
|
};
|
|
if (system) baseParams["system"] = system;
|
|
if (thinkingParam) {
|
|
baseParams["thinking"] = thinkingParam;
|
|
// Anthropic requires temperature=1 (or unset) and rejects top_p when thinking is enabled.
|
|
} else {
|
|
if (typeof body.temperature === "number") baseParams["temperature"] = body.temperature;
|
|
if (typeof body.top_p === "number") baseParams["top_p"] = body.top_p;
|
|
}
|
|
if (body.stop !== undefined) {
|
|
baseParams["stop_sequences"] = Array.isArray(body.stop) ? body.stop : [body.stop];
|
|
}
|
|
|
|
if (!stream) {
|
|
try {
|
|
const message = await client.messages.create(
|
|
baseParams as unknown as MessageCreateParamsNonStreaming,
|
|
);
|
|
let text = "";
|
|
let reasoning = "";
|
|
for (const b of message.content ?? []) {
|
|
if (b.type === "text") text += b.text;
|
|
else if (b.type === "thinking") reasoning += (b as { thinking?: string }).thinking ?? "";
|
|
}
|
|
const finishReason = mapAnthropicStopReason(message.stop_reason ?? null);
|
|
const assistantMessage: Record<string, unknown> = { role: "assistant", content: text };
|
|
if (reasoning) assistantMessage["reasoning_content"] = reasoning;
|
|
res.json({
|
|
id: message.id,
|
|
object: "chat.completion",
|
|
created: Math.floor(Date.now() / 1000),
|
|
model: message.model,
|
|
choices: [
|
|
{
|
|
index: 0,
|
|
message: assistantMessage,
|
|
finish_reason: finishReason,
|
|
},
|
|
],
|
|
usage: {
|
|
prompt_tokens: message.usage?.input_tokens ?? 0,
|
|
completion_tokens: message.usage?.output_tokens ?? 0,
|
|
total_tokens:
|
|
(message.usage?.input_tokens ?? 0) + (message.usage?.output_tokens ?? 0),
|
|
},
|
|
});
|
|
} catch (err) {
|
|
handleProviderError(req, res, err, "anthropic");
|
|
}
|
|
return;
|
|
}
|
|
|
|
let keepalive: NodeJS.Timeout | undefined;
|
|
let clientGone = false;
|
|
res.on("close", () => {
|
|
if (!res.writableEnded) clientGone = true;
|
|
if (keepalive) clearInterval(keepalive);
|
|
});
|
|
|
|
const id = newId();
|
|
const created = Math.floor(Date.now() / 1000);
|
|
|
|
try {
|
|
const upstream = client.messages.stream(
|
|
baseParams as Parameters<typeof client.messages.stream>[0],
|
|
);
|
|
|
|
keepalive = setupSse(res);
|
|
|
|
res.write(
|
|
`data: ${JSON.stringify({
|
|
id,
|
|
object: "chat.completion.chunk",
|
|
created,
|
|
model: body.model,
|
|
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
|
|
})}\n\n`,
|
|
);
|
|
flush(res);
|
|
|
|
let stopReason: string | null = null;
|
|
|
|
for await (const event of upstream) {
|
|
if (clientGone) break;
|
|
if (event.type === "content_block_delta") {
|
|
let delta: Record<string, unknown> | null = null;
|
|
if (event.delta.type === "text_delta") {
|
|
delta = { content: event.delta.text };
|
|
} else if (event.delta.type === "thinking_delta") {
|
|
const td = event.delta as unknown as { thinking?: string };
|
|
if (td.thinking) delta = { reasoning_content: td.thinking };
|
|
}
|
|
if (delta) {
|
|
const chunk = {
|
|
id,
|
|
object: "chat.completion.chunk",
|
|
created,
|
|
model: body.model,
|
|
choices: [{ index: 0, delta, finish_reason: null }],
|
|
};
|
|
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
|
flush(res);
|
|
}
|
|
} else if (event.type === "message_delta") {
|
|
const md = event as unknown as { delta?: { stop_reason?: string | null } };
|
|
if (md.delta?.stop_reason) {
|
|
stopReason = md.delta.stop_reason;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!clientGone) {
|
|
const finishChunk = {
|
|
id,
|
|
object: "chat.completion.chunk",
|
|
created,
|
|
model: body.model,
|
|
choices: [
|
|
{
|
|
index: 0,
|
|
delta: {},
|
|
finish_reason: mapAnthropicStopReason(stopReason),
|
|
},
|
|
],
|
|
};
|
|
res.write(`data: ${JSON.stringify(finishChunk)}\n\n`);
|
|
res.write("data: [DONE]\n\n");
|
|
flush(res);
|
|
}
|
|
} catch (err) {
|
|
if (!res.headersSent) {
|
|
handleProviderError(req, res, err, "anthropic");
|
|
return;
|
|
}
|
|
req.log?.error({ err }, "anthropic stream error");
|
|
try {
|
|
res.write(
|
|
`data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`,
|
|
);
|
|
flush(res);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
} finally {
|
|
if (keepalive) clearInterval(keepalive);
|
|
if (!clientGone) {
|
|
try {
|
|
res.end();
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function mapAnthropicStopReason(reason: string | null): string {
|
|
switch (reason) {
|
|
case "end_turn":
|
|
case "stop_sequence":
|
|
return "stop";
|
|
case "max_tokens":
|
|
return "length";
|
|
case "tool_use":
|
|
return "tool_calls";
|
|
default:
|
|
return "stop";
|
|
}
|
|
}
|
|
|
|
function errorMessage(err: unknown): string {
|
|
if (err instanceof Error) return err.message;
|
|
if (typeof err === "string") return err;
|
|
try {
|
|
return JSON.stringify(err);
|
|
} catch {
|
|
return "unknown error";
|
|
}
|
|
}
|
|
|
|
function providerNotConfigured(res: Response, provider: "openai" | "anthropic"): void {
|
|
res.status(503).json({
|
|
error: {
|
|
message: `${provider} provider is not configured. Pass the matching Replit AI Integrations CLI flags, not public provider endpoints.`,
|
|
type: "server_error",
|
|
},
|
|
});
|
|
}
|
|
|
|
function handleProviderError(
|
|
req: Request,
|
|
res: Response,
|
|
err: unknown,
|
|
provider: string,
|
|
): void {
|
|
req.log?.error({ err, provider }, "provider error");
|
|
const status =
|
|
typeof (err as { status?: number })?.status === "number"
|
|
? (err as { status: number }).status
|
|
: 500;
|
|
if (!res.headersSent) {
|
|
res.status(status).json({
|
|
error: {
|
|
message: errorMessage(err),
|
|
type: "upstream_error",
|
|
provider,
|
|
},
|
|
});
|
|
}
|
|
}
|
|
|
|
export default router;
|