Improve streaming by correctly detecting client disconnects
Externalize OpenAI and Anthropic SDKs and fix streaming logic by using `res.on("close")` instead of `req.on("close")` to accurately detect client disconnects and prevent premature loop termination.
Replit-Commit-Author: Agent
Replit-Commit-Session-Id: dcd090ca-935a-42ae-8db4-503e1bbe165a
Replit-Commit-Checkpoint-Type: full_checkpoint
Replit-Commit-Event-Id: e74b2f37-ad8b-469c-8d06-fdc381ac4a39
Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/40608c50-a053-47e8-aa03-1bdcc1a2698b/dcd090ca-935a-42ae-8db4-503e1bbe165a/ZBSB1hW
Replit-Helium-Checkpoint-Created: true
This commit is contained in:
@@ -29,6 +29,8 @@ async function buildAll() {
|
|||||||
// - use path traversal to read files (e.g. @google-cloud/secret-manager loads sibling .proto files)
|
// - use path traversal to read files (e.g. @google-cloud/secret-manager loads sibling .proto files)
|
||||||
external: [
|
external: [
|
||||||
"*.node",
|
"*.node",
|
||||||
|
"openai",
|
||||||
|
"@anthropic-ai/sdk",
|
||||||
"sharp",
|
"sharp",
|
||||||
"better-sqlite3",
|
"better-sqlite3",
|
||||||
"sqlite3",
|
"sqlite3",
|
||||||
|
|||||||
@@ -153,9 +153,9 @@ async function handleOpenAI(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let keepalive: NodeJS.Timeout | undefined;
|
let keepalive: NodeJS.Timeout | undefined;
|
||||||
let closed = false;
|
let clientGone = false;
|
||||||
req.on("close", () => {
|
res.on("close", () => {
|
||||||
closed = true;
|
if (!res.writableEnded) clientGone = true;
|
||||||
if (keepalive) clearInterval(keepalive);
|
if (keepalive) clearInterval(keepalive);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -168,12 +168,12 @@ async function handleOpenAI(
|
|||||||
keepalive = setupSse(res);
|
keepalive = setupSse(res);
|
||||||
|
|
||||||
for await (const chunk of upstream as AsyncIterable<unknown>) {
|
for await (const chunk of upstream as AsyncIterable<unknown>) {
|
||||||
if (closed) break;
|
if (clientGone) break;
|
||||||
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
|
||||||
res.flush?.();
|
res.flush?.();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!closed) {
|
if (!clientGone) {
|
||||||
res.write("data: [DONE]\n\n");
|
res.write("data: [DONE]\n\n");
|
||||||
res.flush?.();
|
res.flush?.();
|
||||||
}
|
}
|
||||||
@@ -193,7 +193,7 @@ async function handleOpenAI(
|
|||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (keepalive) clearInterval(keepalive);
|
if (keepalive) clearInterval(keepalive);
|
||||||
if (!closed) {
|
if (!clientGone) {
|
||||||
try {
|
try {
|
||||||
res.end();
|
res.end();
|
||||||
} catch {
|
} catch {
|
||||||
@@ -292,9 +292,9 @@ async function handleAnthropic(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let keepalive: NodeJS.Timeout | undefined;
|
let keepalive: NodeJS.Timeout | undefined;
|
||||||
let closed = false;
|
let clientGone = false;
|
||||||
req.on("close", () => {
|
res.on("close", () => {
|
||||||
closed = true;
|
if (!res.writableEnded) clientGone = true;
|
||||||
if (keepalive) clearInterval(keepalive);
|
if (keepalive) clearInterval(keepalive);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -322,7 +322,7 @@ async function handleAnthropic(
|
|||||||
let stopReason: string | null = null;
|
let stopReason: string | null = null;
|
||||||
|
|
||||||
for await (const event of upstream) {
|
for await (const event of upstream) {
|
||||||
if (closed) break;
|
if (clientGone) break;
|
||||||
if (
|
if (
|
||||||
event.type === "content_block_delta" &&
|
event.type === "content_block_delta" &&
|
||||||
event.delta.type === "text_delta"
|
event.delta.type === "text_delta"
|
||||||
@@ -350,7 +350,7 @@ async function handleAnthropic(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!closed) {
|
if (!clientGone) {
|
||||||
const finishChunk = {
|
const finishChunk = {
|
||||||
id,
|
id,
|
||||||
object: "chat.completion.chunk",
|
object: "chat.completion.chunk",
|
||||||
@@ -384,7 +384,7 @@ async function handleAnthropic(
|
|||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (keepalive) clearInterval(keepalive);
|
if (keepalive) clearInterval(keepalive);
|
||||||
if (!closed) {
|
if (!clientGone) {
|
||||||
try {
|
try {
|
||||||
res.end();
|
res.end();
|
||||||
} catch {
|
} catch {
|
||||||
|
|||||||
Reference in New Issue
Block a user