* fix(telegram): handle Grammy HttpError network failures (#3815) Grammy wraps fetch errors in an .error property (not .cause). Added .error traversal to collectErrorCandidates in network-errors.ts. Registered scoped unhandled rejection handler in monitorTelegramProvider to catch network errors that escape the polling loop (e.g., from setMyCommands during bot setup). Handler is unregistered when the provider stops. * fix(telegram): address review feedback for Grammy HttpError handling - Gate .error traversal on HttpError name to avoid widening search graph - Use runtime logger instead of console.warn for consistency - Add isGrammyHttpError check to scope unhandled rejection handler - Consolidate isNetworkRelatedError into isRecoverableTelegramNetworkError - Add 'timeout' to recoverable message snippets for full coverage
This commit is contained in:
parent
f9fae2c439
commit
99b4f2a24e
3 changed files with 164 additions and 111 deletions
|
|
@ -6,6 +6,7 @@ import { loadConfig } from "../config/config.js";
|
||||||
import { computeBackoff, sleepWithAbort } from "../infra/backoff.js";
|
import { computeBackoff, sleepWithAbort } from "../infra/backoff.js";
|
||||||
import { formatErrorMessage } from "../infra/errors.js";
|
import { formatErrorMessage } from "../infra/errors.js";
|
||||||
import { formatDurationMs } from "../infra/format-duration.js";
|
import { formatDurationMs } from "../infra/format-duration.js";
|
||||||
|
import { registerUnhandledRejectionHandler } from "../infra/unhandled-rejections.js";
|
||||||
import { resolveTelegramAccount } from "./accounts.js";
|
import { resolveTelegramAccount } from "./accounts.js";
|
||||||
import { resolveTelegramAllowedUpdates } from "./allowed-updates.js";
|
import { resolveTelegramAllowedUpdates } from "./allowed-updates.js";
|
||||||
import { createTelegramBot } from "./bot.js";
|
import { createTelegramBot } from "./bot.js";
|
||||||
|
|
@ -78,133 +79,137 @@ const isGetUpdatesConflict = (err: unknown) => {
|
||||||
return haystack.includes("getupdates");
|
return haystack.includes("getupdates");
|
||||||
};
|
};
|
||||||
|
|
||||||
const NETWORK_ERROR_SNIPPETS = [
|
/** Check if error is a Grammy HttpError (used to scope unhandled rejection handling) */
|
||||||
"fetch failed",
|
const isGrammyHttpError = (err: unknown): boolean => {
|
||||||
"network",
|
if (!err || typeof err !== "object") {
|
||||||
"timeout",
|
|
||||||
"socket",
|
|
||||||
"econnreset",
|
|
||||||
"econnrefused",
|
|
||||||
"undici",
|
|
||||||
];
|
|
||||||
|
|
||||||
const isNetworkRelatedError = (err: unknown) => {
|
|
||||||
if (!err) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
const message = formatErrorMessage(err).toLowerCase();
|
return (err as { name?: string }).name === "HttpError";
|
||||||
if (!message) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return NETWORK_ERROR_SNIPPETS.some((snippet) => message.includes(snippet));
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) {
|
export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) {
|
||||||
const cfg = opts.config ?? loadConfig();
|
const log = opts.runtime?.error ?? console.error;
|
||||||
const account = resolveTelegramAccount({
|
|
||||||
cfg,
|
|
||||||
accountId: opts.accountId,
|
|
||||||
});
|
|
||||||
const token = opts.token?.trim() || account.token;
|
|
||||||
if (!token) {
|
|
||||||
throw new Error(
|
|
||||||
`Telegram bot token missing for account "${account.accountId}" (set channels.telegram.accounts.${account.accountId}.botToken/tokenFile or TELEGRAM_BOT_TOKEN for default).`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const proxyFetch =
|
// Register handler for Grammy HttpError unhandled rejections.
|
||||||
opts.proxyFetch ?? (account.config.proxy ? makeProxyFetch(account.config.proxy) : undefined);
|
// This catches network errors that escape the polling loop's try-catch
|
||||||
|
// (e.g., from setMyCommands during bot setup).
|
||||||
let lastUpdateId = await readTelegramUpdateOffset({
|
// We gate on isGrammyHttpError to avoid suppressing non-Telegram errors.
|
||||||
accountId: account.accountId,
|
const unregisterHandler = registerUnhandledRejectionHandler((err) => {
|
||||||
});
|
if (isGrammyHttpError(err) && isRecoverableTelegramNetworkError(err, { context: "polling" })) {
|
||||||
const persistUpdateId = async (updateId: number) => {
|
log(`[telegram] Suppressed network error: ${formatErrorMessage(err)}`);
|
||||||
if (lastUpdateId !== null && updateId <= lastUpdateId) {
|
return true; // handled - don't crash
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
lastUpdateId = updateId;
|
return false;
|
||||||
try {
|
});
|
||||||
await writeTelegramUpdateOffset({
|
|
||||||
accountId: account.accountId,
|
try {
|
||||||
updateId,
|
const cfg = opts.config ?? loadConfig();
|
||||||
});
|
const account = resolveTelegramAccount({
|
||||||
} catch (err) {
|
cfg,
|
||||||
(opts.runtime?.error ?? console.error)(
|
accountId: opts.accountId,
|
||||||
`telegram: failed to persist update offset: ${String(err)}`,
|
});
|
||||||
|
const token = opts.token?.trim() || account.token;
|
||||||
|
if (!token) {
|
||||||
|
throw new Error(
|
||||||
|
`Telegram bot token missing for account "${account.accountId}" (set channels.telegram.accounts.${account.accountId}.botToken/tokenFile or TELEGRAM_BOT_TOKEN for default).`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
const bot = createTelegramBot({
|
const proxyFetch =
|
||||||
token,
|
opts.proxyFetch ?? (account.config.proxy ? makeProxyFetch(account.config.proxy) : undefined);
|
||||||
runtime: opts.runtime,
|
|
||||||
proxyFetch,
|
|
||||||
config: cfg,
|
|
||||||
accountId: account.accountId,
|
|
||||||
updateOffset: {
|
|
||||||
lastUpdateId,
|
|
||||||
onUpdateId: persistUpdateId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (opts.useWebhook) {
|
let lastUpdateId = await readTelegramUpdateOffset({
|
||||||
await startTelegramWebhook({
|
|
||||||
token,
|
|
||||||
accountId: account.accountId,
|
accountId: account.accountId,
|
||||||
config: cfg,
|
|
||||||
path: opts.webhookPath,
|
|
||||||
port: opts.webhookPort,
|
|
||||||
secret: opts.webhookSecret,
|
|
||||||
runtime: opts.runtime as RuntimeEnv,
|
|
||||||
fetch: proxyFetch,
|
|
||||||
abortSignal: opts.abortSignal,
|
|
||||||
publicUrl: opts.webhookUrl,
|
|
||||||
});
|
});
|
||||||
return;
|
const persistUpdateId = async (updateId: number) => {
|
||||||
}
|
if (lastUpdateId !== null && updateId <= lastUpdateId) {
|
||||||
|
return;
|
||||||
// Use grammyjs/runner for concurrent update processing
|
}
|
||||||
let restartAttempts = 0;
|
lastUpdateId = updateId;
|
||||||
|
try {
|
||||||
while (!opts.abortSignal?.aborted) {
|
await writeTelegramUpdateOffset({
|
||||||
const runner = run(bot, createTelegramRunnerOptions(cfg));
|
accountId: account.accountId,
|
||||||
const stopOnAbort = () => {
|
updateId,
|
||||||
if (opts.abortSignal?.aborted) {
|
});
|
||||||
void runner.stop();
|
} catch (err) {
|
||||||
|
(opts.runtime?.error ?? console.error)(
|
||||||
|
`telegram: failed to persist update offset: ${String(err)}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
opts.abortSignal?.addEventListener("abort", stopOnAbort, { once: true });
|
|
||||||
try {
|
const bot = createTelegramBot({
|
||||||
// runner.task() returns a promise that resolves when the runner stops
|
token,
|
||||||
await runner.task();
|
runtime: opts.runtime,
|
||||||
|
proxyFetch,
|
||||||
|
config: cfg,
|
||||||
|
accountId: account.accountId,
|
||||||
|
updateOffset: {
|
||||||
|
lastUpdateId,
|
||||||
|
onUpdateId: persistUpdateId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (opts.useWebhook) {
|
||||||
|
await startTelegramWebhook({
|
||||||
|
token,
|
||||||
|
accountId: account.accountId,
|
||||||
|
config: cfg,
|
||||||
|
path: opts.webhookPath,
|
||||||
|
port: opts.webhookPort,
|
||||||
|
secret: opts.webhookSecret,
|
||||||
|
runtime: opts.runtime as RuntimeEnv,
|
||||||
|
fetch: proxyFetch,
|
||||||
|
abortSignal: opts.abortSignal,
|
||||||
|
publicUrl: opts.webhookUrl,
|
||||||
|
});
|
||||||
return;
|
return;
|
||||||
} catch (err) {
|
|
||||||
if (opts.abortSignal?.aborted) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
const isConflict = isGetUpdatesConflict(err);
|
|
||||||
const isRecoverable = isRecoverableTelegramNetworkError(err, { context: "polling" });
|
|
||||||
const isNetworkError = isNetworkRelatedError(err);
|
|
||||||
if (!isConflict && !isRecoverable && !isNetworkError) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
restartAttempts += 1;
|
|
||||||
const delayMs = computeBackoff(TELEGRAM_POLL_RESTART_POLICY, restartAttempts);
|
|
||||||
const reason = isConflict ? "getUpdates conflict" : "network error";
|
|
||||||
const errMsg = formatErrorMessage(err);
|
|
||||||
(opts.runtime?.error ?? console.error)(
|
|
||||||
`Telegram ${reason}: ${errMsg}; retrying in ${formatDurationMs(delayMs)}.`,
|
|
||||||
);
|
|
||||||
try {
|
|
||||||
await sleepWithAbort(delayMs, opts.abortSignal);
|
|
||||||
} catch (sleepErr) {
|
|
||||||
if (opts.abortSignal?.aborted) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
throw sleepErr;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
opts.abortSignal?.removeEventListener("abort", stopOnAbort);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Use grammyjs/runner for concurrent update processing
|
||||||
|
let restartAttempts = 0;
|
||||||
|
|
||||||
|
while (!opts.abortSignal?.aborted) {
|
||||||
|
const runner = run(bot, createTelegramRunnerOptions(cfg));
|
||||||
|
const stopOnAbort = () => {
|
||||||
|
if (opts.abortSignal?.aborted) {
|
||||||
|
void runner.stop();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
opts.abortSignal?.addEventListener("abort", stopOnAbort, { once: true });
|
||||||
|
try {
|
||||||
|
// runner.task() returns a promise that resolves when the runner stops
|
||||||
|
await runner.task();
|
||||||
|
return;
|
||||||
|
} catch (err) {
|
||||||
|
if (opts.abortSignal?.aborted) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
const isConflict = isGetUpdatesConflict(err);
|
||||||
|
const isRecoverable = isRecoverableTelegramNetworkError(err, { context: "polling" });
|
||||||
|
if (!isConflict && !isRecoverable) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
restartAttempts += 1;
|
||||||
|
const delayMs = computeBackoff(TELEGRAM_POLL_RESTART_POLICY, restartAttempts);
|
||||||
|
const reason = isConflict ? "getUpdates conflict" : "network error";
|
||||||
|
const errMsg = formatErrorMessage(err);
|
||||||
|
(opts.runtime?.error ?? console.error)(
|
||||||
|
`Telegram ${reason}: ${errMsg}; retrying in ${formatDurationMs(delayMs)}.`,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await sleepWithAbort(delayMs, opts.abortSignal);
|
||||||
|
} catch (sleepErr) {
|
||||||
|
if (opts.abortSignal?.aborted) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw sleepErr;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
opts.abortSignal?.removeEventListener("abort", stopOnAbort);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
unregisterHandler();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -39,4 +39,43 @@ describe("isRecoverableTelegramNetworkError", () => {
|
||||||
it("returns false for unrelated errors", () => {
|
it("returns false for unrelated errors", () => {
|
||||||
expect(isRecoverableTelegramNetworkError(new Error("invalid token"))).toBe(false);
|
expect(isRecoverableTelegramNetworkError(new Error("invalid token"))).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Grammy HttpError tests (issue #3815)
|
||||||
|
// Grammy wraps fetch errors in .error property, not .cause
|
||||||
|
describe("Grammy HttpError", () => {
|
||||||
|
class MockHttpError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly error: unknown,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "HttpError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it("detects network error wrapped in HttpError", () => {
|
||||||
|
const fetchError = new TypeError("fetch failed");
|
||||||
|
const httpError = new MockHttpError(
|
||||||
|
"Network request for 'setMyCommands' failed!",
|
||||||
|
fetchError,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(isRecoverableTelegramNetworkError(httpError)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("detects network error with cause wrapped in HttpError", () => {
|
||||||
|
const cause = Object.assign(new Error("socket hang up"), { code: "ECONNRESET" });
|
||||||
|
const fetchError = Object.assign(new TypeError("fetch failed"), { cause });
|
||||||
|
const httpError = new MockHttpError("Network request for 'getUpdates' failed!", fetchError);
|
||||||
|
|
||||||
|
expect(isRecoverableTelegramNetworkError(httpError)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns false for non-network errors wrapped in HttpError", () => {
|
||||||
|
const authError = new Error("Unauthorized: bot token is invalid");
|
||||||
|
const httpError = new MockHttpError("Bad Request: invalid token", authError);
|
||||||
|
|
||||||
|
expect(isRecoverableTelegramNetworkError(httpError)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,7 @@ const RECOVERABLE_MESSAGE_SNIPPETS = [
|
||||||
"client network socket disconnected",
|
"client network socket disconnected",
|
||||||
"socket hang up",
|
"socket hang up",
|
||||||
"getaddrinfo",
|
"getaddrinfo",
|
||||||
|
"timeout", // catch timeout messages not covered by error codes/names
|
||||||
];
|
];
|
||||||
|
|
||||||
function normalizeCode(code?: string): string {
|
function normalizeCode(code?: string): string {
|
||||||
|
|
@ -97,6 +98,14 @@ function collectErrorCandidates(err: unknown): unknown[] {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Grammy's HttpError wraps the underlying error in .error (not .cause)
|
||||||
|
// Only follow .error for HttpError to avoid widening the search graph
|
||||||
|
if (getErrorName(current) === "HttpError") {
|
||||||
|
const wrappedError = (current as { error?: unknown }).error;
|
||||||
|
if (wrappedError && !seen.has(wrappedError)) {
|
||||||
|
queue.push(wrappedError);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue