Initial random proxy server
This commit is contained in:
+38
@@ -0,0 +1,38 @@
|
||||
import express, { type Express } from "express";
|
||||
import cors from "cors";
|
||||
import pinoHttp from "pino-http";
|
||||
import dashboardRouter from "./routes/dashboard";
|
||||
import router from "./routes";
|
||||
import proxyRouter from "./routes/proxy";
|
||||
import { logger } from "./lib/logger";
|
||||
|
||||
const app: Express = express();
|
||||
|
||||
app.use(
|
||||
pinoHttp({
|
||||
logger,
|
||||
serializers: {
|
||||
req(req) {
|
||||
return {
|
||||
id: req.id,
|
||||
method: req.method,
|
||||
url: req.url?.split("?")[0],
|
||||
};
|
||||
},
|
||||
res(res) {
|
||||
return {
|
||||
statusCode: res.statusCode,
|
||||
};
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
app.use(cors());
|
||||
app.use(express.json({ limit: "50mb" }));
|
||||
app.use(express.urlencoded({ extended: true, limit: "50mb" }));
|
||||
|
||||
app.use(dashboardRouter);
|
||||
app.use("/api", router);
|
||||
app.use("/v1", proxyRouter);
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,15 @@
|
||||
import app from "./app";
|
||||
import { config } from "./lib/config";
|
||||
import { logger } from "./lib/logger";
|
||||
|
||||
const port = config.server.port;
|
||||
const host = config.server.host;
|
||||
|
||||
app.listen(port, host, (err) => {
|
||||
if (err) {
|
||||
logger.error({ err }, "Error listening on port");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
logger.info({ host, port }, "Server listening");
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
export interface ProviderConfig {
|
||||
baseURL?: string;
|
||||
apiKey?: string;
|
||||
}
|
||||
|
||||
export interface AppConfig {
|
||||
server: {
|
||||
host: string;
|
||||
port: number;
|
||||
};
|
||||
proxy: {
|
||||
apiKey: string;
|
||||
};
|
||||
providers: {
|
||||
openai: ProviderConfig;
|
||||
anthropic: ProviderConfig;
|
||||
};
|
||||
logging: {
|
||||
level: string;
|
||||
pretty: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: AppConfig = {
|
||||
server: {
|
||||
host: "127.0.0.1",
|
||||
port: 8080,
|
||||
},
|
||||
proxy: {
|
||||
apiKey: "dev-proxy-key",
|
||||
},
|
||||
providers: {
|
||||
openai: {},
|
||||
anthropic: {},
|
||||
},
|
||||
logging: {
|
||||
level: "info",
|
||||
pretty: true,
|
||||
},
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | undefined {
|
||||
return typeof value === "string" && value.length > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
function booleanValue(value: unknown): boolean | undefined {
|
||||
return typeof value === "boolean" ? value : undefined;
|
||||
}
|
||||
|
||||
function numberValue(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function providerConfig(value: unknown): ProviderConfig {
|
||||
if (!isRecord(value)) return {};
|
||||
return {
|
||||
...(stringValue(value["baseURL"]) ? { baseURL: stringValue(value["baseURL"]) } : {}),
|
||||
...(stringValue(value["apiKey"]) ? { apiKey: stringValue(value["apiKey"]) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function mergeConfig(raw: unknown): AppConfig {
|
||||
if (!isRecord(raw)) return DEFAULT_CONFIG;
|
||||
|
||||
const server = isRecord(raw["server"]) ? raw["server"] : {};
|
||||
const proxy = isRecord(raw["proxy"]) ? raw["proxy"] : {};
|
||||
const providers = isRecord(raw["providers"]) ? raw["providers"] : {};
|
||||
const logging = isRecord(raw["logging"]) ? raw["logging"] : {};
|
||||
|
||||
const port = numberValue(server["port"]) ?? DEFAULT_CONFIG.server.port;
|
||||
if (!Number.isInteger(port) || port <= 0) {
|
||||
throw new Error(`Invalid config value "server.port": ${port}`);
|
||||
}
|
||||
|
||||
return {
|
||||
server: {
|
||||
host: stringValue(server["host"]) ?? DEFAULT_CONFIG.server.host,
|
||||
port,
|
||||
},
|
||||
proxy: {
|
||||
apiKey: stringValue(proxy["apiKey"]) ?? DEFAULT_CONFIG.proxy.apiKey,
|
||||
},
|
||||
providers: {
|
||||
openai: providerConfig(providers["openai"]),
|
||||
anthropic: providerConfig(providers["anthropic"]),
|
||||
},
|
||||
logging: {
|
||||
level: stringValue(logging["level"]) ?? DEFAULT_CONFIG.logging.level,
|
||||
pretty: booleanValue(logging["pretty"]) ?? DEFAULT_CONFIG.logging.pretty,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function configPathFromArgv(): string {
|
||||
const configArgIndex = process.argv.findIndex((arg) => arg === "--config");
|
||||
if (configArgIndex >= 0) {
|
||||
const value = process.argv[configArgIndex + 1];
|
||||
if (!value) {
|
||||
throw new Error("Missing value after --config");
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
const inlineArg = process.argv.find((arg) => arg.startsWith("--config="));
|
||||
if (inlineArg) {
|
||||
return path.resolve(inlineArg.slice("--config=".length));
|
||||
}
|
||||
|
||||
return path.resolve("config.json");
|
||||
}
|
||||
|
||||
function loadConfig(): AppConfig {
|
||||
const filePath = configPathFromArgv();
|
||||
if (!existsSync(filePath)) {
|
||||
return DEFAULT_CONFIG;
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(filePath, "utf8"));
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
throw new Error(`Could not read config file ${filePath}: ${message}`);
|
||||
}
|
||||
|
||||
return mergeConfig(parsed);
|
||||
}
|
||||
|
||||
export const config = loadConfig();
|
||||
@@ -0,0 +1,19 @@
|
||||
import pino from "pino";
|
||||
import { config } from "./config";
|
||||
|
||||
export const logger = pino({
|
||||
level: config.logging.level,
|
||||
redact: [
|
||||
"req.headers.authorization",
|
||||
"req.headers.cookie",
|
||||
"res.headers['set-cookie']",
|
||||
],
|
||||
...(config.logging.pretty
|
||||
? {
|
||||
transport: {
|
||||
target: "pino-pretty",
|
||||
options: { colorize: true },
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
export const REASONING_BASE_MODELS = [
|
||||
{ id: "gpt-5.2", provider: "openai" as const },
|
||||
{ id: "gpt-5-mini", provider: "openai" as const },
|
||||
{ id: "gpt-5-nano", provider: "openai" as const },
|
||||
{ id: "o4-mini", provider: "openai" as const },
|
||||
{ id: "o3", provider: "openai" as const },
|
||||
{ id: "claude-opus-4-7", provider: "anthropic" as const },
|
||||
{ id: "claude-opus-4-6", provider: "anthropic" as const },
|
||||
{ id: "claude-sonnet-4-6", provider: "anthropic" as const },
|
||||
{ id: "claude-haiku-4-5", provider: "anthropic" as const },
|
||||
];
|
||||
|
||||
export const EFFORT_LEVELS = ["minimal", "low", "medium", "high"] as const;
|
||||
export type EffortLevel = (typeof EFFORT_LEVELS)[number];
|
||||
|
||||
export interface ModelEntry {
|
||||
id: string;
|
||||
provider: "openai" | "anthropic";
|
||||
baseModel: string;
|
||||
effort: EffortLevel | null;
|
||||
}
|
||||
|
||||
export const MODELS: ModelEntry[] = REASONING_BASE_MODELS.flatMap((m) => {
|
||||
const variants: ModelEntry[] = [
|
||||
{ id: m.id, provider: m.provider, baseModel: m.id, effort: null },
|
||||
];
|
||||
for (const effort of EFFORT_LEVELS) {
|
||||
variants.push({
|
||||
id: `${m.id}-${effort}`,
|
||||
provider: m.provider,
|
||||
baseModel: m.id,
|
||||
effort,
|
||||
});
|
||||
}
|
||||
return variants;
|
||||
});
|
||||
|
||||
export const MODEL_INDEX: Map<string, ModelEntry> = new Map(MODELS.map((m) => [m.id, m]));
|
||||
@@ -0,0 +1,151 @@
|
||||
import { Router, type IRouter } from "express";
|
||||
import { MODELS } from "../lib/models";
|
||||
|
||||
const router: IRouter = Router();
|
||||
|
||||
function escapeHtml(value: string): string {
|
||||
return value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """);
|
||||
}
|
||||
|
||||
function modelRows(): string {
|
||||
return MODELS.map((model) => {
|
||||
const effort = model.effort ?? "default";
|
||||
return `<tr>
|
||||
<td><code>${escapeHtml(model.id)}</code></td>
|
||||
<td>${escapeHtml(model.provider)}</td>
|
||||
<td><code>${escapeHtml(model.baseModel)}</code></td>
|
||||
<td>${escapeHtml(effort)}</td>
|
||||
</tr>`;
|
||||
}).join("");
|
||||
}
|
||||
|
||||
function dashboardHtml(): string {
|
||||
return `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>API Server Models</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light dark;
|
||||
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
background: #f7f8fa;
|
||||
color: #1b1f24;
|
||||
}
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
background: #f7f8fa;
|
||||
}
|
||||
main {
|
||||
max-width: 1080px;
|
||||
margin: 0 auto;
|
||||
padding: 32px 20px 48px;
|
||||
}
|
||||
header {
|
||||
display: flex;
|
||||
align-items: end;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-size: 28px;
|
||||
line-height: 1.2;
|
||||
font-weight: 650;
|
||||
}
|
||||
.meta {
|
||||
color: #59616c;
|
||||
font-size: 14px;
|
||||
}
|
||||
.table-wrap {
|
||||
overflow-x: auto;
|
||||
border: 1px solid #d9dee7;
|
||||
border-radius: 8px;
|
||||
background: #ffffff;
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
min-width: 720px;
|
||||
}
|
||||
th, td {
|
||||
padding: 12px 14px;
|
||||
border-bottom: 1px solid #e8ebf0;
|
||||
text-align: left;
|
||||
font-size: 14px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
th {
|
||||
background: #eef1f5;
|
||||
color: #3c434d;
|
||||
font-size: 12px;
|
||||
letter-spacing: 0;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
tr:last-child td {
|
||||
border-bottom: 0;
|
||||
}
|
||||
code {
|
||||
font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
|
||||
font-size: 13px;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root, body {
|
||||
background: #101316;
|
||||
color: #f1f4f8;
|
||||
}
|
||||
.meta {
|
||||
color: #aab3bf;
|
||||
}
|
||||
.table-wrap {
|
||||
background: #171b20;
|
||||
border-color: #303741;
|
||||
}
|
||||
th {
|
||||
background: #20262d;
|
||||
color: #c1cad6;
|
||||
}
|
||||
th, td {
|
||||
border-bottom-color: #29313a;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<h1>Model List</h1>
|
||||
<div class="meta">${MODELS.length} models</div>
|
||||
</header>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Model</th>
|
||||
<th>Provider</th>
|
||||
<th>Base Model</th>
|
||||
<th>Effort</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${modelRows()}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
router.get(["/", "/dashboard"], (_req, res) => {
|
||||
res.type("html").send(dashboardHtml());
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Router, type IRouter } from "express";
|
||||
|
||||
const router: IRouter = Router();
|
||||
|
||||
router.get("/healthz", (_req, res) => {
|
||||
res.json({ status: "ok" });
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,8 @@
|
||||
import { Router, type IRouter } from "express";
|
||||
import healthRouter from "./health";
|
||||
|
||||
const router: IRouter = Router();
|
||||
|
||||
router.use(healthRouter);
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,525 @@
|
||||
import { Router, type IRouter, type Request, type Response } from "express";
|
||||
import OpenAI from "openai";
|
||||
import Anthropic from "@anthropic-ai/sdk";
|
||||
import type { MessageCreateParamsNonStreaming } from "@anthropic-ai/sdk/resources/messages";
|
||||
import { config } from "../lib/config";
|
||||
import { MODEL_INDEX, MODELS } from "../lib/models";
|
||||
|
||||
const router: IRouter = Router();
|
||||
|
||||
function openAIClient(): OpenAI | null {
|
||||
const provider = config.providers.openai;
|
||||
if (!provider.apiKey || !provider.baseURL) return null;
|
||||
return new OpenAI({
|
||||
baseURL: provider.baseURL,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
}
|
||||
|
||||
function anthropicClient(): Anthropic | null {
|
||||
const provider = config.providers.anthropic;
|
||||
if (!provider.apiKey || !provider.baseURL) return null;
|
||||
return new Anthropic({
|
||||
baseURL: provider.baseURL,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
}
|
||||
|
||||
function flush(res: Response): void {
|
||||
(res as Response & { flush?: () => void }).flush?.();
|
||||
}
|
||||
|
||||
function authorize(req: Request, res: Response): boolean {
|
||||
const expected = config.proxy.apiKey;
|
||||
const header = req.header("authorization") ?? "";
|
||||
const match = header.match(/^Bearer\s+(.+)$/i);
|
||||
if (!match || match[1] !== expected) {
|
||||
res.status(401).json({ error: { message: "Invalid API key", type: "invalid_request_error" } });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
router.get("/models", (req, res) => {
|
||||
if (!authorize(req, res)) return;
|
||||
const created = Math.floor(Date.now() / 1000);
|
||||
res.json({
|
||||
object: "list",
|
||||
data: MODELS.map((m) => ({
|
||||
id: m.id,
|
||||
object: "model",
|
||||
created,
|
||||
owned_by: m.provider,
|
||||
})),
|
||||
});
|
||||
});
|
||||
|
||||
interface ChatMessage {
|
||||
role: "system" | "user" | "assistant" | "developer" | "tool";
|
||||
content: unknown;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
interface ChatCompletionRequest {
|
||||
model: string;
|
||||
messages: ChatMessage[];
|
||||
stream?: boolean;
|
||||
temperature?: number;
|
||||
top_p?: number;
|
||||
max_tokens?: number;
|
||||
max_completion_tokens?: number;
|
||||
stop?: string | string[];
|
||||
frequency_penalty?: number;
|
||||
presence_penalty?: number;
|
||||
user?: string;
|
||||
reasoning_effort?: "minimal" | "low" | "medium" | "high";
|
||||
thinking?: { type?: string; budget_tokens?: number } | null;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
function effortToBudget(effort: string | undefined): number | null {
|
||||
switch (effort) {
|
||||
case "minimal":
|
||||
return 1024;
|
||||
case "low":
|
||||
return 2048;
|
||||
case "medium":
|
||||
return 8000;
|
||||
case "high":
|
||||
return 16000;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isOpenAIModel(model: string): boolean {
|
||||
return /^(gpt|o\d|chatgpt)/i.test(model);
|
||||
}
|
||||
|
||||
function isAnthropicModel(model: string): boolean {
|
||||
return /^claude/i.test(model);
|
||||
}
|
||||
|
||||
function newId(): string {
|
||||
return "chatcmpl-" + Math.random().toString(36).slice(2, 14);
|
||||
}
|
||||
|
||||
router.post("/chat/completions", async (req, res) => {
|
||||
if (!authorize(req, res)) return;
|
||||
|
||||
const body = req.body as ChatCompletionRequest;
|
||||
if (!body || typeof body !== "object" || !body.model || !Array.isArray(body.messages)) {
|
||||
res.status(400).json({
|
||||
error: { message: "model and messages are required", type: "invalid_request_error" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const requestedId = body.model;
|
||||
const entry = MODEL_INDEX.get(requestedId);
|
||||
|
||||
let provider: "openai" | "anthropic";
|
||||
if (entry) {
|
||||
body.model = entry.baseModel;
|
||||
if (entry.effort && !body.reasoning_effort) {
|
||||
body.reasoning_effort = entry.effort;
|
||||
}
|
||||
provider = entry.provider;
|
||||
} else if (isOpenAIModel(requestedId)) {
|
||||
provider = "openai";
|
||||
} else if (isAnthropicModel(requestedId)) {
|
||||
provider = "anthropic";
|
||||
} else {
|
||||
res.status(400).json({
|
||||
error: {
|
||||
message: `Unsupported model: ${requestedId}`,
|
||||
type: "invalid_request_error",
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = body.stream === true;
|
||||
|
||||
if (provider === "openai") {
|
||||
await handleOpenAI(req, res, body, stream);
|
||||
} else {
|
||||
await handleAnthropic(req, res, body, stream);
|
||||
}
|
||||
});
|
||||
|
||||
function setupSse(res: Response): NodeJS.Timeout {
|
||||
res.setHeader("Content-Type", "text/event-stream");
|
||||
res.setHeader("Cache-Control", "no-cache, no-transform");
|
||||
res.setHeader("Connection", "keep-alive");
|
||||
res.setHeader("X-Accel-Buffering", "no");
|
||||
res.flushHeaders();
|
||||
const keepalive = setInterval(() => {
|
||||
try {
|
||||
res.write(": keepalive\n\n");
|
||||
flush(res);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}, 5000);
|
||||
return keepalive;
|
||||
}
|
||||
|
||||
async function handleOpenAI(
|
||||
req: Request,
|
||||
res: Response,
|
||||
body: ChatCompletionRequest,
|
||||
stream: boolean,
|
||||
): Promise<void> {
|
||||
const client = openAIClient();
|
||||
if (!client) {
|
||||
providerNotConfigured(res, "openai");
|
||||
return;
|
||||
}
|
||||
|
||||
const { stream: _ignored, ...rest } = body;
|
||||
|
||||
if (!stream) {
|
||||
try {
|
||||
const completion = await client.chat.completions.create({
|
||||
...rest,
|
||||
stream: false,
|
||||
} as Parameters<typeof client.chat.completions.create>[0]);
|
||||
res.json(completion);
|
||||
} catch (err) {
|
||||
handleProviderError(req, res, err, "openai");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let keepalive: NodeJS.Timeout | undefined;
|
||||
let clientGone = false;
|
||||
res.on("close", () => {
|
||||
if (!res.writableEnded) clientGone = true;
|
||||
if (keepalive) clearInterval(keepalive);
|
||||
});
|
||||
|
||||
try {
|
||||
const upstream = await client.chat.completions.create({
|
||||
...rest,
|
||||
stream: true,
|
||||
} as Parameters<typeof client.chat.completions.create>[0]);
|
||||
|
||||
keepalive = setupSse(res);
|
||||
|
||||
for await (const chunk of upstream as AsyncIterable<unknown>) {
|
||||
if (clientGone) break;
|
||||
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
||||
flush(res);
|
||||
}
|
||||
|
||||
if (!clientGone) {
|
||||
res.write("data: [DONE]\n\n");
|
||||
flush(res);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!res.headersSent) {
|
||||
handleProviderError(req, res, err, "openai");
|
||||
return;
|
||||
}
|
||||
req.log?.error({ err }, "openai stream error");
|
||||
try {
|
||||
res.write(
|
||||
`data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`,
|
||||
);
|
||||
flush(res);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
} finally {
|
||||
if (keepalive) clearInterval(keepalive);
|
||||
if (!clientGone) {
|
||||
try {
|
||||
res.end();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface AnthropicConvertedMessages {
|
||||
system: string | undefined;
|
||||
messages: Array<{ role: "user" | "assistant"; content: unknown }>;
|
||||
}
|
||||
|
||||
function convertMessagesForAnthropic(messages: ChatMessage[]): AnthropicConvertedMessages {
|
||||
const systemParts: string[] = [];
|
||||
const out: Array<{ role: "user" | "assistant"; content: unknown }> = [];
|
||||
for (const msg of messages) {
|
||||
if (msg.role === "system" || msg.role === "developer") {
|
||||
if (typeof msg.content === "string") {
|
||||
systemParts.push(msg.content);
|
||||
} else if (Array.isArray(msg.content)) {
|
||||
for (const part of msg.content as Array<{ type?: string; text?: string }>) {
|
||||
if (part?.type === "text" && typeof part.text === "string") {
|
||||
systemParts.push(part.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (msg.role === "user" || msg.role === "assistant") {
|
||||
out.push({ role: msg.role, content: msg.content });
|
||||
}
|
||||
// ignore tool messages for this passthrough
|
||||
}
|
||||
return {
|
||||
system: systemParts.length > 0 ? systemParts.join("\n\n") : undefined,
|
||||
messages: out,
|
||||
};
|
||||
}
|
||||
|
||||
async function handleAnthropic(
|
||||
req: Request,
|
||||
res: Response,
|
||||
body: ChatCompletionRequest,
|
||||
stream: boolean,
|
||||
): Promise<void> {
|
||||
const client = anthropicClient();
|
||||
if (!client) {
|
||||
providerNotConfigured(res, "anthropic");
|
||||
return;
|
||||
}
|
||||
|
||||
const { system, messages } = convertMessagesForAnthropic(body.messages);
|
||||
let maxTokens =
|
||||
body.max_completion_tokens ??
|
||||
body.max_tokens ??
|
||||
8192;
|
||||
|
||||
let thinkingParam: { type: "enabled"; budget_tokens: number } | null = null;
|
||||
if (body.thinking && (body.thinking.type === "enabled" || typeof body.thinking.budget_tokens === "number")) {
|
||||
const budget = Math.max(1024, body.thinking.budget_tokens ?? 4096);
|
||||
thinkingParam = { type: "enabled", budget_tokens: budget };
|
||||
} else {
|
||||
const budget = effortToBudget(body.reasoning_effort);
|
||||
if (budget !== null) {
|
||||
thinkingParam = { type: "enabled", budget_tokens: budget };
|
||||
}
|
||||
}
|
||||
|
||||
if (thinkingParam) {
|
||||
if (maxTokens <= thinkingParam.budget_tokens) {
|
||||
maxTokens = thinkingParam.budget_tokens + 4096;
|
||||
}
|
||||
}
|
||||
|
||||
const baseParams: Record<string, unknown> = {
|
||||
model: body.model,
|
||||
max_tokens: maxTokens,
|
||||
messages,
|
||||
};
|
||||
if (system) baseParams["system"] = system;
|
||||
if (thinkingParam) {
|
||||
baseParams["thinking"] = thinkingParam;
|
||||
// Anthropic requires temperature=1 (or unset) and rejects top_p when thinking is enabled.
|
||||
} else {
|
||||
if (typeof body.temperature === "number") baseParams["temperature"] = body.temperature;
|
||||
if (typeof body.top_p === "number") baseParams["top_p"] = body.top_p;
|
||||
}
|
||||
if (body.stop !== undefined) {
|
||||
baseParams["stop_sequences"] = Array.isArray(body.stop) ? body.stop : [body.stop];
|
||||
}
|
||||
|
||||
if (!stream) {
|
||||
try {
|
||||
const message = await client.messages.create(
|
||||
baseParams as unknown as MessageCreateParamsNonStreaming,
|
||||
);
|
||||
let text = "";
|
||||
let reasoning = "";
|
||||
for (const b of message.content ?? []) {
|
||||
if (b.type === "text") text += b.text;
|
||||
else if (b.type === "thinking") reasoning += (b as { thinking?: string }).thinking ?? "";
|
||||
}
|
||||
const finishReason = mapAnthropicStopReason(message.stop_reason ?? null);
|
||||
const assistantMessage: Record<string, unknown> = { role: "assistant", content: text };
|
||||
if (reasoning) assistantMessage["reasoning_content"] = reasoning;
|
||||
res.json({
|
||||
id: message.id,
|
||||
object: "chat.completion",
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
model: message.model,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: assistantMessage,
|
||||
finish_reason: finishReason,
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
prompt_tokens: message.usage?.input_tokens ?? 0,
|
||||
completion_tokens: message.usage?.output_tokens ?? 0,
|
||||
total_tokens:
|
||||
(message.usage?.input_tokens ?? 0) + (message.usage?.output_tokens ?? 0),
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
handleProviderError(req, res, err, "anthropic");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let keepalive: NodeJS.Timeout | undefined;
|
||||
let clientGone = false;
|
||||
res.on("close", () => {
|
||||
if (!res.writableEnded) clientGone = true;
|
||||
if (keepalive) clearInterval(keepalive);
|
||||
});
|
||||
|
||||
const id = newId();
|
||||
const created = Math.floor(Date.now() / 1000);
|
||||
|
||||
try {
|
||||
const upstream = client.messages.stream(
|
||||
baseParams as Parameters<typeof client.messages.stream>[0],
|
||||
);
|
||||
|
||||
keepalive = setupSse(res);
|
||||
|
||||
res.write(
|
||||
`data: ${JSON.stringify({
|
||||
id,
|
||||
object: "chat.completion.chunk",
|
||||
created,
|
||||
model: body.model,
|
||||
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
|
||||
})}\n\n`,
|
||||
);
|
||||
flush(res);
|
||||
|
||||
let stopReason: string | null = null;
|
||||
|
||||
for await (const event of upstream) {
|
||||
if (clientGone) break;
|
||||
if (event.type === "content_block_delta") {
|
||||
let delta: Record<string, unknown> | null = null;
|
||||
if (event.delta.type === "text_delta") {
|
||||
delta = { content: event.delta.text };
|
||||
} else if (event.delta.type === "thinking_delta") {
|
||||
const td = event.delta as unknown as { thinking?: string };
|
||||
if (td.thinking) delta = { reasoning_content: td.thinking };
|
||||
}
|
||||
if (delta) {
|
||||
const chunk = {
|
||||
id,
|
||||
object: "chat.completion.chunk",
|
||||
created,
|
||||
model: body.model,
|
||||
choices: [{ index: 0, delta, finish_reason: null }],
|
||||
};
|
||||
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
||||
flush(res);
|
||||
}
|
||||
} else if (event.type === "message_delta") {
|
||||
const md = event as unknown as { delta?: { stop_reason?: string | null } };
|
||||
if (md.delta?.stop_reason) {
|
||||
stopReason = md.delta.stop_reason;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!clientGone) {
|
||||
const finishChunk = {
|
||||
id,
|
||||
object: "chat.completion.chunk",
|
||||
created,
|
||||
model: body.model,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: {},
|
||||
finish_reason: mapAnthropicStopReason(stopReason),
|
||||
},
|
||||
],
|
||||
};
|
||||
res.write(`data: ${JSON.stringify(finishChunk)}\n\n`);
|
||||
res.write("data: [DONE]\n\n");
|
||||
flush(res);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!res.headersSent) {
|
||||
handleProviderError(req, res, err, "anthropic");
|
||||
return;
|
||||
}
|
||||
req.log?.error({ err }, "anthropic stream error");
|
||||
try {
|
||||
res.write(
|
||||
`data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`,
|
||||
);
|
||||
flush(res);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
} finally {
|
||||
if (keepalive) clearInterval(keepalive);
|
||||
if (!clientGone) {
|
||||
try {
|
||||
res.end();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function mapAnthropicStopReason(reason: string | null): string {
|
||||
switch (reason) {
|
||||
case "end_turn":
|
||||
case "stop_sequence":
|
||||
return "stop";
|
||||
case "max_tokens":
|
||||
return "length";
|
||||
case "tool_use":
|
||||
return "tool_calls";
|
||||
default:
|
||||
return "stop";
|
||||
}
|
||||
}
|
||||
|
||||
function errorMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
if (typeof err === "string") return err;
|
||||
try {
|
||||
return JSON.stringify(err);
|
||||
} catch {
|
||||
return "unknown error";
|
||||
}
|
||||
}
|
||||
|
||||
function providerNotConfigured(res: Response, provider: "openai" | "anthropic"): void {
|
||||
res.status(503).json({
|
||||
error: {
|
||||
message: `${provider} provider is not configured. Add providers.${provider}.apiKey and providers.${provider}.baseURL to config.json. Use the internal endpoint supplied by your runtime or agent, not the public provider endpoint.`,
|
||||
type: "server_error",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function handleProviderError(
|
||||
req: Request,
|
||||
res: Response,
|
||||
err: unknown,
|
||||
provider: string,
|
||||
): void {
|
||||
req.log?.error({ err, provider }, "provider error");
|
||||
const status =
|
||||
typeof (err as { status?: number })?.status === "number"
|
||||
? (err as { status: number }).status
|
||||
: 500;
|
||||
if (!res.headersSent) {
|
||||
res.status(status).json({
|
||||
error: {
|
||||
message: errorMessage(err),
|
||||
type: "upstream_error",
|
||||
provider,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export default router;
|
||||
Reference in New Issue
Block a user