commit 8c68a3004451435b1ce6fcbb2b2cc3dc69aa17f0 Author: hwhung Date: Thu May 7 19:13:04 2026 +0800 Initial proxy artifact diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8349eff --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +dist/ +node_modules/ +.pnpm-store/ +config.json +config.local.json +pnpm-lock.yaml diff --git a/.replit b/.replit new file mode 100644 index 0000000..254524c --- /dev/null +++ b/.replit @@ -0,0 +1,2 @@ +run = "PORT=${PORT:-3000} PROXY_API_KEY=${PROXY_API_KEY:-dev-proxy-key} pnpm prod" +onBoot = "pnpm install --ignore-workspace" diff --git a/.replit-artifact/artifact.toml b/.replit-artifact/artifact.toml new file mode 100644 index 0000000..9066ada --- /dev/null +++ b/.replit-artifact/artifact.toml @@ -0,0 +1,23 @@ +kind = "web" +previewPath = "/" +title = "Proxy Artifact" +version = "1.0.0" +id = "proxy-artifact" +router = "path" + +[[services]] +name = "web" +paths = ["/"] +localPort = 3000 + +[services.development] +run = "PORT=${PORT:-3000} PROXY_API_KEY=${PROXY_API_KEY:-dev-proxy-key} pnpm prod" + +[services.production] +build = ["sh", "-c", "pnpm install --ignore-workspace && pnpm build:fast"] +run = ["sh", "-c", "PORT=${PORT:-3000} PROXY_API_KEY=${PROXY_API_KEY:-dev-proxy-key} NODE_ENV=production node --enable-source-maps dist/index.mjs"] + +[services.env] +PORT = "3000" +PROXY_API_KEY = "dev-proxy-key" +BASE_PATH = "/" diff --git a/build.mjs b/build.mjs new file mode 100644 index 0000000..111c94c --- /dev/null +++ b/build.mjs @@ -0,0 +1,128 @@ +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { build as esbuild } from "esbuild"; +import esbuildPluginPino from "esbuild-plugin-pino"; +import { rm } from "node:fs/promises"; + +// Plugins (e.g. 'esbuild-plugin-pino') may use `require` to resolve dependencies +globalThis.require = createRequire(import.meta.url); + +const artifactDir = path.dirname(fileURLToPath(import.meta.url)); + +async function buildAll() { + const distDir = path.resolve(artifactDir, "dist"); + await rm(distDir, { recursive: true, force: true }); + + await esbuild({ + entryPoints: [path.resolve(artifactDir, "src/index.ts")], + platform: "node", + bundle: true, + format: "esm", + outdir: distDir, + outExtension: { ".js": ".mjs" }, + logLevel: "info", + // Some packages may not be bundleable, so we externalize them, we can add more here as needed. + // Some of the packages below may not be imported or installed, but we're adding them in case they are in the future. + // Examples of unbundleable packages: + // - uses native modules and loads them dynamically (e.g. sharp) + // - use path traversal to read files (e.g. @google-cloud/secret-manager loads sibling .proto files) + external: [ + "*.node", + "openai", + "@anthropic-ai/sdk", + "sharp", + "better-sqlite3", + "sqlite3", + "canvas", + "bcrypt", + "argon2", + "fsevents", + "re2", + "farmhash", + "xxhash-addon", + "bufferutil", + "utf-8-validate", + "ssh2", + "cpu-features", + "dtrace-provider", + "isolated-vm", + "lightningcss", + "pg-native", + "oracledb", + "mongodb-client-encryption", + "nodemailer", + "handlebars", + "knex", + "typeorm", + "protobufjs", + "onnxruntime-node", + "@tensorflow/*", + "@prisma/client", + "@mikro-orm/*", + "@grpc/*", + "@swc/*", + "@aws-sdk/*", + "@azure/*", + "@opentelemetry/*", + "@google-cloud/*", + "@google/*", + "googleapis", + "firebase-admin", + "@parcel/watcher", + "@sentry/profiling-node", + "@tree-sitter/*", + "aws-sdk", + "classic-level", + "dd-trace", + "ffi-napi", + "grpc", + "hiredis", + "kerberos", + "leveldown", + "miniflare", + "mysql2", + "newrelic", + "odbc", + "piscina", + "realm", + "ref-napi", + "rocksdb", + "sass-embedded", + "sequelize", + "serialport", + "snappy", + "tinypool", + "usb", + "workerd", + "wrangler", + "zeromq", + "zeromq-prebuilt", + "playwright", + "puppeteer", + "puppeteer-core", + "electron", + ], + sourcemap: "linked", + plugins: [ + // pino relies on workers to handle logging, instead of externalizing it we use a plugin to handle it + esbuildPluginPino({ transports: ["pino-pretty"] }) + ], + // Make sure packages that are cjs only (e.g. express) but are bundled continue to work in our esm output file + banner: { + js: `import { createRequire as __bannerCrReq } from 'node:module'; +import __bannerPath from 'node:path'; +import __bannerUrl from 'node:url'; + +globalThis.require = __bannerCrReq(import.meta.url); +globalThis.__filename = __bannerUrl.fileURLToPath(import.meta.url); +globalThis.__dirname = __bannerPath.dirname(globalThis.__filename); + `, + }, + }); +} + +buildAll().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..0520e30 --- /dev/null +++ b/package.json @@ -0,0 +1,35 @@ +{ + "name": "proxy-artifact", + "version": "0.0.0", + "license": "MIT", + "type": "module", + "scripts": { + "dev": "tsx src/index.ts", + "build": "tsc --noEmit && node build.mjs", + "build:fast": "node build.mjs", + "prod": "node build.mjs && node --enable-source-maps dist/index.mjs", + "start": "node --enable-source-maps dist/index.mjs", + "typecheck": "tsc --noEmit" + }, + "private": true, + "dependencies": { + "@anthropic-ai/sdk": "^0.91.1", + "cors": "^2.8.6", + "express": "^5.2.1", + "openai": "^6.34.0", + "pino": "^9.14.0", + "pino-http": "^10.5.0" + }, + "devDependencies": { + "@types/cors": "^2.8.19", + "@types/express": "^5.0.6", + "@types/node": "^25.3.5", + "esbuild": "0.27.3", + "esbuild-plugin-pino": "^2.3.3", + "pino-pretty": "^13.1.3", + "prettier": "^3.8.1", + "thread-stream": "3.1.0", + "tsx": "^4.21.0", + "typescript": "~5.9.2" + } +} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..ccbac80 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +packages: + - "." diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..48cc35d --- /dev/null +++ b/readme.md @@ -0,0 +1,53 @@ +# Proxy Artifact + +Standalone TypeScript API proxy packaged as a Replit `kind = "web"` artifact. + +## Replit + +The repo includes: + +- `.replit` for the Run button +- `.replit-artifact/artifact.toml` for Replit Publish + +Run command: + +```bash +PORT=${PORT:-3000} PROXY_API_KEY=${PROXY_API_KEY:-dev-proxy-key} pnpm prod +``` + +Production publish build: + +```bash +pnpm install --ignore-workspace && pnpm build:fast +``` + +Production publish run: + +```bash +PORT=${PORT:-3000} PROXY_API_KEY=${PROXY_API_KEY:-dev-proxy-key} NODE_ENV=production node --enable-source-maps dist/index.mjs +``` + +## Config + +The app reads Replit AI Integration variables automatically: + +- `AI_INTEGRATIONS_OPENAI_BASE_URL` +- `AI_INTEGRATIONS_OPENAI_API_KEY` +- `AI_INTEGRATIONS_ANTHROPIC_BASE_URL` +- `AI_INTEGRATIONS_ANTHROPIC_API_KEY` + +Server/proxy overrides: + +- `HOST` +- `PORT` +- `PROXY_API_KEY` + +CLI flags still work for manual runs, but mass deployments should use environment variables. + +## Commands + +- `pnpm install --ignore-workspace` +- `pnpm prod` +- `pnpm run build:fast` +- `pnpm run build` +- `pnpm run typecheck` diff --git a/replit.md b/replit.md new file mode 100644 index 0000000..37135d7 --- /dev/null +++ b/replit.md @@ -0,0 +1,59 @@ +# Workspace + +## Overview + +Standalone TypeScript API server packaged as a Replit `kind = "web"` artifact. + +## Stack + +- **Node.js version**: 24 +- **Package manager**: pnpm +- **TypeScript version**: 5.9 +- **API framework**: Express 5 +- **Build**: esbuild + +## Provider Endpoints + +Use Replit AI Integrations endpoints. The app reads these environment variables automatically: + +- `AI_INTEGRATIONS_OPENAI_BASE_URL` +- `AI_INTEGRATIONS_OPENAI_API_KEY` +- `AI_INTEGRATIONS_ANTHROPIC_BASE_URL` +- `AI_INTEGRATIONS_ANTHROPIC_API_KEY` + +CLI flags are still available for manual overrides: + +- `--openai-base-url` +- `--openai-api-key` +- `--anthropic-base-url` +- `--anthropic-api-key` + +Never use public official provider endpoints. + +## Key Commands + +- `pnpm install --ignore-workspace` — install dependencies without joining a parent workspace +- `pnpm prod` — build and run API server locally on port `3000` +- `pnpm run typecheck` — typecheck the server +- `pnpm run build` — typecheck and build `dist/index.mjs` +- `pnpm run build:fast` — build `dist/index.mjs` without typechecking +- `pnpm start` — run the built server + +## Replit Configuration + +- `.replit` `run` uses `pnpm prod` +- `.replit` `onBoot` uses `pnpm install --ignore-workspace` +- `.replit-artifact/artifact.toml` marks this repo as a publishable `kind = "web"` artifact + +## Server Overrides + +- `--host` or `HOST` +- `--port` or `PORT` +- `--proxy-api-key` or `PROXY_API_KEY` + +## Routes + +- `/` and `/dashboard` — static model dashboard +- `/api/healthz` — health check +- `/v1/models` — OpenAI-compatible model list +- `/v1/chat/completions` — OpenAI-compatible chat proxy diff --git a/setup.md b/setup.md new file mode 100644 index 0000000..4fbdf94 --- /dev/null +++ b/setup.md @@ -0,0 +1,55 @@ +Clone and run https://git.komnenos.me/springfw/proxy-artifact in ./proxy-artifact. + +Goal: get the publishable web artifact running once on port 3000 with OpenAI and Anthropic +Replit AI Integrations wired. + +Bearer token for this deployment: DEPLOYMENT_GENERATED_PROXY_TOKEN + +Important: wire OpenAI and Anthropic via the ai-integrations-openai and +ai-integrations-anthropic skills. Call setupReplitAIIntegrations directly. Do not call +searchIntegrations. OpenAI and Anthropic are Replit AI Integrations that provide +AI_INTEGRATIONS_* variables; they are not OAuth connectors. + +Do not: +- add it to a parent workspace +- move it into artifacts/ +- modify source files +- edit package.json +- create config.json or /tmp config files +- inspect/free unrelated ports +- use nohup/setsid/background shell tricks +- call searchIntegrations +- search the Replit integrations catalog +- create provider API secrets manually +- use public OpenAI or Anthropic endpoints +- invent, rotate, print, store, or replace the bearer token +- run a foreground server command directly in the shell +- run verification commands +- run chat-completion smoke tests unless asked + +Steps: + +1. Clone into ./proxy-artifact. + +2. Wire OpenAI and Anthropic via the ai-integrations-openai and ai-integrations-anthropic +skills: + - call setupReplitAIIntegrations directly + - do not call searchIntegrations + +3. From ./proxy-artifact, run: + pnpm install --ignore-workspace + +4. Ensure ./proxy-artifact/.replit-artifact/artifact.toml remains kind = "web". + +5. Use the existing artifact run command, but set PROXY_API_KEY to DEPLOYMENT_GENERATED_PROXY_TOKEN +for this deployment. + +6. Start/restart the artifact/workflow once. + +7. Do not run health, dashboard, model-list, or chat-completion verification commands. + +Report only: +- local URL +- whether setupReplitAIIntegrations wired OpenAI and Anthropic +- whether the kind = "web" artifact was used +- that no verification commands were run diff --git a/src/app.ts b/src/app.ts new file mode 100644 index 0000000..6daa56b --- /dev/null +++ b/src/app.ts @@ -0,0 +1,38 @@ +import express, { type Express } from "express"; +import cors from "cors"; +import pinoHttp from "pino-http"; +import dashboardRouter from "./routes/dashboard"; +import router from "./routes"; +import proxyRouter from "./routes/proxy"; +import { logger } from "./lib/logger"; + +const app: Express = express(); + +app.use( + pinoHttp({ + logger, + serializers: { + req(req) { + return { + id: req.id, + method: req.method, + url: req.url?.split("?")[0], + }; + }, + res(res) { + return { + statusCode: res.statusCode, + }; + }, + }, + }), +); +app.use(cors()); +app.use(express.json({ limit: "50mb" })); +app.use(express.urlencoded({ extended: true, limit: "50mb" })); + +app.use(dashboardRouter); +app.use("/api", router); +app.use("/v1", proxyRouter); + +export default app; diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..fce85ae --- /dev/null +++ b/src/index.ts @@ -0,0 +1,15 @@ +import app from "./app"; +import { config } from "./lib/config"; +import { logger } from "./lib/logger"; + +const port = config.server.port; +const host = config.server.host; + +app.listen(port, host, (err) => { + if (err) { + logger.error({ err }, "Error listening on port"); + process.exit(1); + } + + logger.info({ host, port }, "Server listening"); +}); diff --git a/src/lib/config.ts b/src/lib/config.ts new file mode 100644 index 0000000..4e4d7b7 --- /dev/null +++ b/src/lib/config.ts @@ -0,0 +1,110 @@ +export interface ProviderConfig { + baseURL?: string; + apiKey?: string; +} + +export interface AppConfig { + server: { + host: string; + port: number; + }; + proxy: { + apiKey: string; + }; + providers: { + openai: ProviderConfig; + anthropic: ProviderConfig; + }; + logging: { + level: string; + pretty: boolean; + }; +} + +const DEFAULT_CONFIG: AppConfig = { + server: { + host: "0.0.0.0", + port: 3000, + }, + proxy: { + apiKey: "dev-proxy-key", + }, + providers: { + openai: {}, + anthropic: {}, + }, + logging: { + level: "info", + pretty: true, + }, +}; + +function argValue(name: string): string | undefined { + const longName = `--${name}`; + const index = process.argv.findIndex((arg) => arg === longName); + if (index >= 0) { + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; + } + + const inlineArg = process.argv.find((arg) => arg.startsWith(`${longName}=`)); + return inlineArg ? inlineArg.slice(longName.length + 1) : undefined; +} + +function argFlag(name: string): boolean { + return process.argv.includes(`--${name}`); +} + +function envValue(name: string): string | undefined { + const value = process.env[name]; + return value && value.length > 0 ? value : undefined; +} + +function applyArgOverrides(base: AppConfig): AppConfig { + const rawPort = argValue("port") ?? envValue("PORT"); + const port = rawPort === undefined ? base.server.port : Number(rawPort); + if (!Number.isInteger(port) || port <= 0) { + throw new Error(`Invalid port value: ${rawPort}`); + } + + const host = argValue("host") ?? envValue("HOST") ?? base.server.host; + const proxyApiKey = argValue("proxy-api-key") ?? envValue("PROXY_API_KEY") ?? base.proxy.apiKey; + const openaiBaseURL = argValue("openai-base-url") ?? envValue("AI_INTEGRATIONS_OPENAI_BASE_URL"); + const openaiApiKey = argValue("openai-api-key") ?? envValue("AI_INTEGRATIONS_OPENAI_API_KEY"); + const anthropicBaseURL = + argValue("anthropic-base-url") ?? envValue("AI_INTEGRATIONS_ANTHROPIC_BASE_URL"); + const anthropicApiKey = + argValue("anthropic-api-key") ?? envValue("AI_INTEGRATIONS_ANTHROPIC_API_KEY"); + + return { + server: { + host, + port, + }, + proxy: { + apiKey: proxyApiKey, + }, + providers: { + openai: { + ...base.providers.openai, + ...(openaiBaseURL ? { baseURL: openaiBaseURL } : {}), + ...(openaiApiKey ? { apiKey: openaiApiKey } : {}), + }, + anthropic: { + ...base.providers.anthropic, + ...(anthropicBaseURL ? { baseURL: anthropicBaseURL } : {}), + ...(anthropicApiKey ? { apiKey: anthropicApiKey } : {}), + }, + }, + logging: { + level: argValue("log-level") ?? base.logging.level, + pretty: argFlag("pretty-logs") + ? true + : argFlag("no-pretty-logs") + ? false + : base.logging.pretty, + }, + }; +} + +export const config = applyArgOverrides(DEFAULT_CONFIG); diff --git a/src/lib/logger.ts b/src/lib/logger.ts new file mode 100644 index 0000000..e11bd0b --- /dev/null +++ b/src/lib/logger.ts @@ -0,0 +1,19 @@ +import pino from "pino"; +import { config } from "./config"; + +export const logger = pino({ + level: config.logging.level, + redact: [ + "req.headers.authorization", + "req.headers.cookie", + "res.headers['set-cookie']", + ], + ...(config.logging.pretty + ? { + transport: { + target: "pino-pretty", + options: { colorize: true }, + }, + } + : {}), +}); diff --git a/src/lib/models.ts b/src/lib/models.ts new file mode 100644 index 0000000..94bf01d --- /dev/null +++ b/src/lib/models.ts @@ -0,0 +1,38 @@ +export const REASONING_BASE_MODELS = [ + { id: "gpt-5.2", provider: "openai" as const }, + { id: "gpt-5-mini", provider: "openai" as const }, + { id: "gpt-5-nano", provider: "openai" as const }, + { id: "o4-mini", provider: "openai" as const }, + { id: "o3", provider: "openai" as const }, + { id: "claude-opus-4-7", provider: "anthropic" as const }, + { id: "claude-opus-4-6", provider: "anthropic" as const }, + { id: "claude-sonnet-4-6", provider: "anthropic" as const }, + { id: "claude-haiku-4-5", provider: "anthropic" as const }, +]; + +export const EFFORT_LEVELS = ["minimal", "low", "medium", "high"] as const; +export type EffortLevel = (typeof EFFORT_LEVELS)[number]; + +export interface ModelEntry { + id: string; + provider: "openai" | "anthropic"; + baseModel: string; + effort: EffortLevel | null; +} + +export const MODELS: ModelEntry[] = REASONING_BASE_MODELS.flatMap((m) => { + const variants: ModelEntry[] = [ + { id: m.id, provider: m.provider, baseModel: m.id, effort: null }, + ]; + for (const effort of EFFORT_LEVELS) { + variants.push({ + id: `${m.id}-${effort}`, + provider: m.provider, + baseModel: m.id, + effort, + }); + } + return variants; +}); + +export const MODEL_INDEX: Map = new Map(MODELS.map((m) => [m.id, m])); diff --git a/src/routes/dashboard.ts b/src/routes/dashboard.ts new file mode 100644 index 0000000..73df2af --- /dev/null +++ b/src/routes/dashboard.ts @@ -0,0 +1,151 @@ +import { Router, type IRouter } from "express"; +import { MODELS } from "../lib/models"; + +const router: IRouter = Router(); + +function escapeHtml(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +function modelRows(): string { + return MODELS.map((model) => { + const effort = model.effort ?? "default"; + return ` + ${escapeHtml(model.id)} + ${escapeHtml(model.provider)} + ${escapeHtml(model.baseModel)} + ${escapeHtml(effort)} + `; + }).join(""); +} + +function dashboardHtml(): string { + return ` + + + + + API Server Models + + + +
+
+

Model List

+
${MODELS.length} models
+
+
+ + + + + + + + + + + ${modelRows()} + +
ModelProviderBase ModelEffort
+
+
+ +`; +} + +router.get(["/", "/dashboard"], (_req, res) => { + res.type("html").send(dashboardHtml()); +}); + +export default router; diff --git a/src/routes/health.ts b/src/routes/health.ts new file mode 100644 index 0000000..14ec3c1 --- /dev/null +++ b/src/routes/health.ts @@ -0,0 +1,9 @@ +import { Router, type IRouter } from "express"; + +const router: IRouter = Router(); + +router.get("/healthz", (_req, res) => { + res.json({ status: "ok" }); +}); + +export default router; diff --git a/src/routes/index.ts b/src/routes/index.ts new file mode 100644 index 0000000..5a1f77a --- /dev/null +++ b/src/routes/index.ts @@ -0,0 +1,8 @@ +import { Router, type IRouter } from "express"; +import healthRouter from "./health"; + +const router: IRouter = Router(); + +router.use(healthRouter); + +export default router; diff --git a/src/routes/proxy.ts b/src/routes/proxy.ts new file mode 100644 index 0000000..5240cc9 --- /dev/null +++ b/src/routes/proxy.ts @@ -0,0 +1,525 @@ +import { Router, type IRouter, type Request, type Response } from "express"; +import OpenAI from "openai"; +import Anthropic from "@anthropic-ai/sdk"; +import type { MessageCreateParamsNonStreaming } from "@anthropic-ai/sdk/resources/messages"; +import { config } from "../lib/config"; +import { MODEL_INDEX, MODELS } from "../lib/models"; + +const router: IRouter = Router(); + +function openAIClient(): OpenAI | null { + const provider = config.providers.openai; + if (!provider.apiKey || !provider.baseURL) return null; + return new OpenAI({ + baseURL: provider.baseURL, + apiKey: provider.apiKey, + }); +} + +function anthropicClient(): Anthropic | null { + const provider = config.providers.anthropic; + if (!provider.apiKey || !provider.baseURL) return null; + return new Anthropic({ + baseURL: provider.baseURL, + apiKey: provider.apiKey, + }); +} + +function flush(res: Response): void { + (res as Response & { flush?: () => void }).flush?.(); +} + +function authorize(req: Request, res: Response): boolean { + const expected = config.proxy.apiKey; + const header = req.header("authorization") ?? ""; + const match = header.match(/^Bearer\s+(.+)$/i); + if (!match || match[1] !== expected) { + res.status(401).json({ error: { message: "Invalid API key", type: "invalid_request_error" } }); + return false; + } + return true; +} + +router.get("/models", (req, res) => { + if (!authorize(req, res)) return; + const created = Math.floor(Date.now() / 1000); + res.json({ + object: "list", + data: MODELS.map((m) => ({ + id: m.id, + object: "model", + created, + owned_by: m.provider, + })), + }); +}); + +interface ChatMessage { + role: "system" | "user" | "assistant" | "developer" | "tool"; + content: unknown; + name?: string; +} + +interface ChatCompletionRequest { + model: string; + messages: ChatMessage[]; + stream?: boolean; + temperature?: number; + top_p?: number; + max_tokens?: number; + max_completion_tokens?: number; + stop?: string | string[]; + frequency_penalty?: number; + presence_penalty?: number; + user?: string; + reasoning_effort?: "minimal" | "low" | "medium" | "high"; + thinking?: { type?: string; budget_tokens?: number } | null; + [key: string]: unknown; +} + +function effortToBudget(effort: string | undefined): number | null { + switch (effort) { + case "minimal": + return 1024; + case "low": + return 2048; + case "medium": + return 8000; + case "high": + return 16000; + default: + return null; + } +} + +function isOpenAIModel(model: string): boolean { + return /^(gpt|o\d|chatgpt)/i.test(model); +} + +function isAnthropicModel(model: string): boolean { + return /^claude/i.test(model); +} + +function newId(): string { + return "chatcmpl-" + Math.random().toString(36).slice(2, 14); +} + +router.post("/chat/completions", async (req, res) => { + if (!authorize(req, res)) return; + + const body = req.body as ChatCompletionRequest; + if (!body || typeof body !== "object" || !body.model || !Array.isArray(body.messages)) { + res.status(400).json({ + error: { message: "model and messages are required", type: "invalid_request_error" }, + }); + return; + } + + const requestedId = body.model; + const entry = MODEL_INDEX.get(requestedId); + + let provider: "openai" | "anthropic"; + if (entry) { + body.model = entry.baseModel; + if (entry.effort && !body.reasoning_effort) { + body.reasoning_effort = entry.effort; + } + provider = entry.provider; + } else if (isOpenAIModel(requestedId)) { + provider = "openai"; + } else if (isAnthropicModel(requestedId)) { + provider = "anthropic"; + } else { + res.status(400).json({ + error: { + message: `Unsupported model: ${requestedId}`, + type: "invalid_request_error", + }, + }); + return; + } + + const stream = body.stream === true; + + if (provider === "openai") { + await handleOpenAI(req, res, body, stream); + } else { + await handleAnthropic(req, res, body, stream); + } +}); + +function setupSse(res: Response): NodeJS.Timeout { + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache, no-transform"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders(); + const keepalive = setInterval(() => { + try { + res.write(": keepalive\n\n"); + flush(res); + } catch { + // ignore + } + }, 5000); + return keepalive; +} + +async function handleOpenAI( + req: Request, + res: Response, + body: ChatCompletionRequest, + stream: boolean, +): Promise { + const client = openAIClient(); + if (!client) { + providerNotConfigured(res, "openai"); + return; + } + + const { stream: _ignored, ...rest } = body; + + if (!stream) { + try { + const completion = await client.chat.completions.create({ + ...rest, + stream: false, + } as Parameters[0]); + res.json(completion); + } catch (err) { + handleProviderError(req, res, err, "openai"); + } + return; + } + + let keepalive: NodeJS.Timeout | undefined; + let clientGone = false; + res.on("close", () => { + if (!res.writableEnded) clientGone = true; + if (keepalive) clearInterval(keepalive); + }); + + try { + const upstream = await client.chat.completions.create({ + ...rest, + stream: true, + } as Parameters[0]); + + keepalive = setupSse(res); + + for await (const chunk of upstream as AsyncIterable) { + if (clientGone) break; + res.write(`data: ${JSON.stringify(chunk)}\n\n`); + flush(res); + } + + if (!clientGone) { + res.write("data: [DONE]\n\n"); + flush(res); + } + } catch (err) { + if (!res.headersSent) { + handleProviderError(req, res, err, "openai"); + return; + } + req.log?.error({ err }, "openai stream error"); + try { + res.write( + `data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`, + ); + flush(res); + } catch { + // ignore + } + } finally { + if (keepalive) clearInterval(keepalive); + if (!clientGone) { + try { + res.end(); + } catch { + // ignore + } + } + } +} + +interface AnthropicConvertedMessages { + system: string | undefined; + messages: Array<{ role: "user" | "assistant"; content: unknown }>; +} + +function convertMessagesForAnthropic(messages: ChatMessage[]): AnthropicConvertedMessages { + const systemParts: string[] = []; + const out: Array<{ role: "user" | "assistant"; content: unknown }> = []; + for (const msg of messages) { + if (msg.role === "system" || msg.role === "developer") { + if (typeof msg.content === "string") { + systemParts.push(msg.content); + } else if (Array.isArray(msg.content)) { + for (const part of msg.content as Array<{ type?: string; text?: string }>) { + if (part?.type === "text" && typeof part.text === "string") { + systemParts.push(part.text); + } + } + } + } else if (msg.role === "user" || msg.role === "assistant") { + out.push({ role: msg.role, content: msg.content }); + } + // ignore tool messages for this passthrough + } + return { + system: systemParts.length > 0 ? systemParts.join("\n\n") : undefined, + messages: out, + }; +} + +async function handleAnthropic( + req: Request, + res: Response, + body: ChatCompletionRequest, + stream: boolean, +): Promise { + const client = anthropicClient(); + if (!client) { + providerNotConfigured(res, "anthropic"); + return; + } + + const { system, messages } = convertMessagesForAnthropic(body.messages); + let maxTokens = + body.max_completion_tokens ?? + body.max_tokens ?? + 8192; + + let thinkingParam: { type: "enabled"; budget_tokens: number } | null = null; + if (body.thinking && (body.thinking.type === "enabled" || typeof body.thinking.budget_tokens === "number")) { + const budget = Math.max(1024, body.thinking.budget_tokens ?? 4096); + thinkingParam = { type: "enabled", budget_tokens: budget }; + } else { + const budget = effortToBudget(body.reasoning_effort); + if (budget !== null) { + thinkingParam = { type: "enabled", budget_tokens: budget }; + } + } + + if (thinkingParam) { + if (maxTokens <= thinkingParam.budget_tokens) { + maxTokens = thinkingParam.budget_tokens + 4096; + } + } + + const baseParams: Record = { + model: body.model, + max_tokens: maxTokens, + messages, + }; + if (system) baseParams["system"] = system; + if (thinkingParam) { + baseParams["thinking"] = thinkingParam; + // Anthropic requires temperature=1 (or unset) and rejects top_p when thinking is enabled. + } else { + if (typeof body.temperature === "number") baseParams["temperature"] = body.temperature; + if (typeof body.top_p === "number") baseParams["top_p"] = body.top_p; + } + if (body.stop !== undefined) { + baseParams["stop_sequences"] = Array.isArray(body.stop) ? body.stop : [body.stop]; + } + + if (!stream) { + try { + const message = await client.messages.create( + baseParams as unknown as MessageCreateParamsNonStreaming, + ); + let text = ""; + let reasoning = ""; + for (const b of message.content ?? []) { + if (b.type === "text") text += b.text; + else if (b.type === "thinking") reasoning += (b as { thinking?: string }).thinking ?? ""; + } + const finishReason = mapAnthropicStopReason(message.stop_reason ?? null); + const assistantMessage: Record = { role: "assistant", content: text }; + if (reasoning) assistantMessage["reasoning_content"] = reasoning; + res.json({ + id: message.id, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: message.model, + choices: [ + { + index: 0, + message: assistantMessage, + finish_reason: finishReason, + }, + ], + usage: { + prompt_tokens: message.usage?.input_tokens ?? 0, + completion_tokens: message.usage?.output_tokens ?? 0, + total_tokens: + (message.usage?.input_tokens ?? 0) + (message.usage?.output_tokens ?? 0), + }, + }); + } catch (err) { + handleProviderError(req, res, err, "anthropic"); + } + return; + } + + let keepalive: NodeJS.Timeout | undefined; + let clientGone = false; + res.on("close", () => { + if (!res.writableEnded) clientGone = true; + if (keepalive) clearInterval(keepalive); + }); + + const id = newId(); + const created = Math.floor(Date.now() / 1000); + + try { + const upstream = client.messages.stream( + baseParams as Parameters[0], + ); + + keepalive = setupSse(res); + + res.write( + `data: ${JSON.stringify({ + id, + object: "chat.completion.chunk", + created, + model: body.model, + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + })}\n\n`, + ); + flush(res); + + let stopReason: string | null = null; + + for await (const event of upstream) { + if (clientGone) break; + if (event.type === "content_block_delta") { + let delta: Record | null = null; + if (event.delta.type === "text_delta") { + delta = { content: event.delta.text }; + } else if (event.delta.type === "thinking_delta") { + const td = event.delta as unknown as { thinking?: string }; + if (td.thinking) delta = { reasoning_content: td.thinking }; + } + if (delta) { + const chunk = { + id, + object: "chat.completion.chunk", + created, + model: body.model, + choices: [{ index: 0, delta, finish_reason: null }], + }; + res.write(`data: ${JSON.stringify(chunk)}\n\n`); + flush(res); + } + } else if (event.type === "message_delta") { + const md = event as unknown as { delta?: { stop_reason?: string | null } }; + if (md.delta?.stop_reason) { + stopReason = md.delta.stop_reason; + } + } + } + + if (!clientGone) { + const finishChunk = { + id, + object: "chat.completion.chunk", + created, + model: body.model, + choices: [ + { + index: 0, + delta: {}, + finish_reason: mapAnthropicStopReason(stopReason), + }, + ], + }; + res.write(`data: ${JSON.stringify(finishChunk)}\n\n`); + res.write("data: [DONE]\n\n"); + flush(res); + } + } catch (err) { + if (!res.headersSent) { + handleProviderError(req, res, err, "anthropic"); + return; + } + req.log?.error({ err }, "anthropic stream error"); + try { + res.write( + `data: ${JSON.stringify({ error: { message: errorMessage(err), type: "upstream_error" } })}\n\n`, + ); + flush(res); + } catch { + // ignore + } + } finally { + if (keepalive) clearInterval(keepalive); + if (!clientGone) { + try { + res.end(); + } catch { + // ignore + } + } + } +} + +function mapAnthropicStopReason(reason: string | null): string { + switch (reason) { + case "end_turn": + case "stop_sequence": + return "stop"; + case "max_tokens": + return "length"; + case "tool_use": + return "tool_calls"; + default: + return "stop"; + } +} + +function errorMessage(err: unknown): string { + if (err instanceof Error) return err.message; + if (typeof err === "string") return err; + try { + return JSON.stringify(err); + } catch { + return "unknown error"; + } +} + +function providerNotConfigured(res: Response, provider: "openai" | "anthropic"): void { + res.status(503).json({ + error: { + message: `${provider} provider is not configured. Pass the matching Replit AI Integrations CLI flags, not public provider endpoints.`, + type: "server_error", + }, + }); +} + +function handleProviderError( + req: Request, + res: Response, + err: unknown, + provider: string, +): void { + req.log?.error({ err, provider }, "provider error"); + const status = + typeof (err as { status?: number })?.status === "number" + ? (err as { status: number }).status + : 500; + if (!res.headersSent) { + res.status(status).json({ + error: { + message: errorMessage(err), + type: "upstream_error", + provider, + }, + }); + } +} + +export default router; diff --git a/tsconfig.base.json b/tsconfig.base.json new file mode 100644 index 0000000..0d23073 --- /dev/null +++ b/tsconfig.base.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "isolatedModules": true, + "lib": ["es2022"], + "module": "esnext", + "moduleResolution": "bundler", + "noEmitOnError": true, + "noFallthroughCasesInSwitch": true, + "noImplicitOverride": false, + "noImplicitReturns": true, + "noUnusedLocals": false, + "noImplicitAny": true, + "noImplicitThis": true, + "strictNullChecks": true, + "strictFunctionTypes": false, + "strictBindCallApply": true, + "strictPropertyInitialization": true, + "useUnknownInCatchVariables": true, + "alwaysStrict": true, + "skipLibCheck": true, + "target": "es2022", + "types": [], + "customConditions": ["workspace"] + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..6f69d7b --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.base.json", + "compileOnSave": false, + "compilerOptions": { + "types": ["node"] + }, + "include": ["src/**/*.ts"] +}