Add reasoning effort support for Anthropic models
Update the API proxy to handle Anthropic's extended thinking feature, mapping OpenAI's `reasoning_effort` parameter to Anthropic's `thinking.budget_tokens` and streaming thinking content as `reasoning_content`. Replit-Commit-Author: Agent Replit-Commit-Session-Id: dcd090ca-935a-42ae-8db4-503e1bbe165a Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: 0c459609-81b9-48ba-a245-b04cd4ce70fc Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/40608c50-a053-47e8-aa03-1bdcc1a2698b/dcd090ca-935a-42ae-8db4-503e1bbe165a/ZBSB1hW Replit-Helium-Checkpoint-Created: true
This commit is contained in:
@@ -72,9 +72,26 @@ interface ChatCompletionRequest {
|
|||||||
frequency_penalty?: number;
|
frequency_penalty?: number;
|
||||||
presence_penalty?: number;
|
presence_penalty?: number;
|
||||||
user?: string;
|
user?: string;
|
||||||
|
reasoning_effort?: "minimal" | "low" | "medium" | "high";
|
||||||
|
thinking?: { type?: string; budget_tokens?: number } | null;
|
||||||
[key: string]: unknown;
|
[key: string]: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function effortToBudget(effort: string | undefined): number | null {
|
||||||
|
switch (effort) {
|
||||||
|
case "minimal":
|
||||||
|
return 1024;
|
||||||
|
case "low":
|
||||||
|
return 2048;
|
||||||
|
case "medium":
|
||||||
|
return 8000;
|
||||||
|
case "high":
|
||||||
|
return 16000;
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function isOpenAIModel(model: string): boolean {
|
function isOpenAIModel(model: string): boolean {
|
||||||
return /^(gpt|o\d|chatgpt)/i.test(model);
|
return /^(gpt|o\d|chatgpt)/i.test(model);
|
||||||
}
|
}
|
||||||
@@ -240,19 +257,41 @@ async function handleAnthropic(
|
|||||||
stream: boolean,
|
stream: boolean,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const { system, messages } = convertMessagesForAnthropic(body.messages);
|
const { system, messages } = convertMessagesForAnthropic(body.messages);
|
||||||
const maxTokens =
|
let maxTokens =
|
||||||
body.max_completion_tokens ??
|
body.max_completion_tokens ??
|
||||||
body.max_tokens ??
|
body.max_tokens ??
|
||||||
8192;
|
8192;
|
||||||
|
|
||||||
|
let thinkingParam: { type: "enabled"; budget_tokens: number } | null = null;
|
||||||
|
if (body.thinking && (body.thinking.type === "enabled" || typeof body.thinking.budget_tokens === "number")) {
|
||||||
|
const budget = Math.max(1024, body.thinking.budget_tokens ?? 4096);
|
||||||
|
thinkingParam = { type: "enabled", budget_tokens: budget };
|
||||||
|
} else {
|
||||||
|
const budget = effortToBudget(body.reasoning_effort);
|
||||||
|
if (budget !== null) {
|
||||||
|
thinkingParam = { type: "enabled", budget_tokens: budget };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (thinkingParam) {
|
||||||
|
if (maxTokens <= thinkingParam.budget_tokens) {
|
||||||
|
maxTokens = thinkingParam.budget_tokens + 4096;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const baseParams: Record<string, unknown> = {
|
const baseParams: Record<string, unknown> = {
|
||||||
model: body.model,
|
model: body.model,
|
||||||
max_tokens: maxTokens,
|
max_tokens: maxTokens,
|
||||||
messages,
|
messages,
|
||||||
};
|
};
|
||||||
if (system) baseParams["system"] = system;
|
if (system) baseParams["system"] = system;
|
||||||
|
if (thinkingParam) {
|
||||||
|
baseParams["thinking"] = thinkingParam;
|
||||||
|
// Anthropic requires temperature=1 (or unset) and rejects top_p when thinking is enabled.
|
||||||
|
} else {
|
||||||
if (typeof body.temperature === "number") baseParams["temperature"] = body.temperature;
|
if (typeof body.temperature === "number") baseParams["temperature"] = body.temperature;
|
||||||
if (typeof body.top_p === "number") baseParams["top_p"] = body.top_p;
|
if (typeof body.top_p === "number") baseParams["top_p"] = body.top_p;
|
||||||
|
}
|
||||||
if (body.stop !== undefined) {
|
if (body.stop !== undefined) {
|
||||||
baseParams["stop_sequences"] = Array.isArray(body.stop) ? body.stop : [body.stop];
|
baseParams["stop_sequences"] = Array.isArray(body.stop) ? body.stop : [body.stop];
|
||||||
}
|
}
|
||||||
@@ -262,10 +301,15 @@ async function handleAnthropic(
|
|||||||
const message = await anthropic.messages.create(
|
const message = await anthropic.messages.create(
|
||||||
baseParams as Parameters<typeof anthropic.messages.create>[0],
|
baseParams as Parameters<typeof anthropic.messages.create>[0],
|
||||||
);
|
);
|
||||||
const text = (message.content ?? [])
|
let text = "";
|
||||||
.map((b) => (b.type === "text" ? b.text : ""))
|
let reasoning = "";
|
||||||
.join("");
|
for (const b of message.content ?? []) {
|
||||||
|
if (b.type === "text") text += b.text;
|
||||||
|
else if (b.type === "thinking") reasoning += (b as { thinking?: string }).thinking ?? "";
|
||||||
|
}
|
||||||
const finishReason = mapAnthropicStopReason(message.stop_reason ?? null);
|
const finishReason = mapAnthropicStopReason(message.stop_reason ?? null);
|
||||||
|
const assistantMessage: Record<string, unknown> = { role: "assistant", content: text };
|
||||||
|
if (reasoning) assistantMessage["reasoning_content"] = reasoning;
|
||||||
res.json({
|
res.json({
|
||||||
id: message.id,
|
id: message.id,
|
||||||
object: "chat.completion",
|
object: "chat.completion",
|
||||||
@@ -274,7 +318,7 @@ async function handleAnthropic(
|
|||||||
choices: [
|
choices: [
|
||||||
{
|
{
|
||||||
index: 0,
|
index: 0,
|
||||||
message: { role: "assistant", content: text },
|
message: assistantMessage,
|
||||||
finish_reason: finishReason,
|
finish_reason: finishReason,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@@ -323,29 +367,29 @@ async function handleAnthropic(
|
|||||||
|
|
||||||
for await (const event of upstream) {
|
for await (const event of upstream) {
|
||||||
if (clientGone) break;
|
if (clientGone) break;
|
||||||
if (
|
if (event.type === "content_block_delta") {
|
||||||
event.type === "content_block_delta" &&
|
let delta: Record<string, unknown> | null = null;
|
||||||
event.delta.type === "text_delta"
|
if (event.delta.type === "text_delta") {
|
||||||
) {
|
delta = { content: event.delta.text };
|
||||||
|
} else if (event.delta.type === "thinking_delta") {
|
||||||
|
const td = event.delta as unknown as { thinking?: string };
|
||||||
|
if (td.thinking) delta = { reasoning_content: td.thinking };
|
||||||
|
}
|
||||||
|
if (delta) {
|
||||||
const chunk = {
|
const chunk = {
|
||||||
id,
|
id,
|
||||||
object: "chat.completion.chunk",
|
object: "chat.completion.chunk",
|
||||||
created,
|
created,
|
||||||
model: body.model,
|
model: body.model,
|
||||||
choices: [
|
choices: [{ index: 0, delta, finish_reason: null }],
|
||||||
{
|
|
||||||
index: 0,
|
|
||||||
delta: { content: event.delta.text },
|
|
||||||
finish_reason: null,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
};
|
||||||
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
||||||
res.flush?.();
|
res.flush?.();
|
||||||
|
}
|
||||||
} else if (event.type === "message_delta") {
|
} else if (event.type === "message_delta") {
|
||||||
const delta = event as unknown as { delta?: { stop_reason?: string | null } };
|
const md = event as unknown as { delta?: { stop_reason?: string | null } };
|
||||||
if (delta.delta?.stop_reason) {
|
if (md.delta?.stop_reason) {
|
||||||
stopReason = delta.delta.stop_reason;
|
stopReason = md.delta.stop_reason;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user