From f21a400c82e42a2a62904c62be2b3f35bd71d406 Mon Sep 17 00:00:00 2001 From: JamesFlare1212 Date: Thu, 23 Apr 2026 03:06:15 -0400 Subject: [PATCH] fix(auth): prevent cookie loss during remote server timeout storms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server timeouts caused orphaned fetchActivityData calls to fire clearCookieCache() asynchronously, destroying cookies for all concurrent callers. Three fixes: 1. Replace Promise.race timeout with AbortController to properly cancel orphaned fetches and prevent delayed clearCookieCache() calls 2. Add cookie backup/restore — backupCookies() before clearCookieCache(), restoreCookieBackup() if re-login fails, so cookies are never lost 3. Add 15s auth failure throttle to block thundering herd re-logins when server slowdowns generate many 500 errors simultaneously --- engage-api/get-activity.ts | 58 +++++++++++++++++++-------------- services/cache-manager.ts | 33 +++++++++++++++---- services/playwright-auth.ts | 65 +++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 32 deletions(-) diff --git a/engage-api/get-activity.ts b/engage-api/get-activity.ts index dad35d1..182faf8 100644 --- a/engage-api/get-activity.ts +++ b/engage-api/get-activity.ts @@ -1,12 +1,15 @@ // engage-api/get-activity.ts -import axios from 'axios'; +import axios, { type AxiosRequestConfig } from 'axios'; import { logger } from '../utils/logger'; import { ensureSingleLogin, - loadCachedCookies, saveCookiesToCache, clearCookieCache, - getCachedCookieString + getCachedCookieString, + backupCookies, + restoreCookieBackup, + tryAcquireAuthLock, + releaseAuthCooldown } from '../services/playwright-auth'; // Define interfaces for our data structures @@ -51,7 +54,8 @@ async function getActivityDetailsRaw( activityId: string, cookies: string, maxRetries: number = 3, - timeoutMilliseconds: number = 10000 + timeoutMilliseconds: number = 10000, + signal?: AbortSignal ): Promise { const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails'; const headers = { @@ -65,13 +69,17 @@ async function getActivityDetailsRaw( }; for (let attempt = 0; attempt < maxRetries; attempt++) { + if (signal?.aborted) { + logger.debug(`Activity ${activityId} aborted before attempt ${attempt + 1}`); + return null; + } try { logger.debug(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} - Sending POST request to ${url}`); const response = await axios.post(url, payload, { headers, timeout: timeoutMilliseconds, responseType: 'text', - // Add additional timeout safety + signal, maxRedirects: 5 }); @@ -155,6 +163,7 @@ export async function fetchActivityData( userName: string, userPwd: string, forceLogin: boolean = false, + signal?: AbortSignal ): Promise { let currentCookie = forceLogin ? null : await getCachedCookieString(); @@ -164,17 +173,10 @@ export async function fetchActivityData( currentCookie = null; } - // Optimization: Skip pre-validation, directly request data - // Only validate/re-login when we get 4xx error OR after 5xx (backend may be in degraded state) if (!currentCookie) { logger.info('No cached cookie found. Attempting login...'); try { currentCookie = await getCompleteCookies(userName, userPwd); - - const cookies = await loadCachedCookies(); - if (cookies) { - await saveCookiesToCache(cookies); - } } catch (loginError) { logger.error(`Login process failed: ${(loginError as Error).message}`); return null; @@ -187,37 +189,41 @@ export async function fetchActivityData( } - logger.debug('Using cached cookie for API request.'); - + try { logger.debug(`Calling getActivityDetailsRaw for activity ${activityId}...`); - const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie); + const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie, 3, 10000, signal); logger.debug(`getActivityDetailsRaw returned for activity ${activityId}`); if (rawActivityDetailsString) { const parsedOuter = JSON.parse(rawActivityDetailsString); return JSON.parse(parsedOuter.d); } - // Check if this was a 5xx error and set flag for cookie validation logger.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`); return null; } catch (error) { + if (signal?.aborted) { + logger.debug(`Activity ${activityId} fetch aborted.`); + return null; + } if (error instanceof AuthenticationError) { - // Cookie returned 4xx, now validate and re-login - logger.warn(`API returned 4xx error (Status: ${error.status}). Cookie may be invalid. Attempting re-login and retry.`); + // Throttle: prevent thundering herd from multiple 500 errors + if (!tryAcquireAuthLock()) { + logger.info(`Auth throttled for activity ${activityId}. Reusing current cookies — likely still valid.`); + return null; + } + + // Backup cookies before clearing so we can restore on re-login failure + backupCookies(); await clearCookieCache(); try { logger.info('Attempting re-login due to authentication failure...'); currentCookie = await getCompleteCookies(userName, userPwd); - - const cookies = await loadCachedCookies(); - if (cookies) { - await saveCookiesToCache(cookies); - } + releaseAuthCooldown(); logger.info('Re-login successful. Retrying request for activity details...'); - const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie); + const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie, 1, 10000, signal); if (rawActivityDetailsStringRetry) { const parsedOuterRetry = JSON.parse(rawActivityDetailsStringRetry); return JSON.parse(parsedOuterRetry.d); @@ -225,7 +231,9 @@ export async function fetchActivityData( logger.warn(`Still no details for activity ${activityId} after re-login and retry.`); return null; } catch (retryLoginOrFetchError) { - logger.error(`Error during re-login or retry fetch for activity ${activityId}: ${(retryLoginOrFetchError as Error).message}`); + logger.error(`Re-login or retry failed for activity ${activityId}: ${(retryLoginOrFetchError as Error).message}`); + // Restore old cookies instead of leaving cache empty + await restoreCookieBackup(); return null; } } else { diff --git a/services/cache-manager.ts b/services/cache-manager.ts index 5239f9b..683b527 100644 --- a/services/cache-manager.ts +++ b/services/cache-manager.ts @@ -52,14 +52,33 @@ async function processAndCacheActivity(activityId: string, forceUpdate: boolean throw new Error('API username or password not configured'); } - // Add timeout protection for the entire fetch operation + // Add timeout protection via AbortController - properly cancels orphaned fetches logger.debug(`Fetching activity data for ID: ${activityId}`); - const activityJson = await Promise.race([ - fetchActivityData(activityId, USERNAME, PASSWORD, false), - new Promise((_, reject) => - setTimeout(() => reject(new Error(`Timeout fetching activity ${activityId} after ${CRAWLER_REQUEST_TIMEOUT_MS}ms`)), CRAWLER_REQUEST_TIMEOUT_MS + 5000) - ) - ]); + const controller = new AbortController(); + const timeoutId = setTimeout( + () => controller.abort(), + CRAWLER_REQUEST_TIMEOUT_MS + 5000 + ); + + let activityJson: any = null; + try { + activityJson = await fetchActivityData( + activityId, + USERNAME, + PASSWORD, + false, + controller.signal + ); + } finally { + clearTimeout(timeoutId); + } + + if (controller.signal.aborted) { + logger.warn(`Request for activity ${activityId} timed out after ${CRAWLER_REQUEST_TIMEOUT_MS + 5000}ms. Cancelling orphaned fetch.`); + // Preserve existing cache on timeout + const existingData = await getActivityData(activityId); + return existingData || { lastCheck: new Date().toISOString(), error: `Timeout after ${CRAWLER_REQUEST_TIMEOUT_MS + 5000}ms` }; + } let structuredActivity: ActivityData; if (!activityJson) { diff --git a/services/playwright-auth.ts b/services/playwright-auth.ts index e30227b..4c737ae 100644 --- a/services/playwright-auth.ts +++ b/services/playwright-auth.ts @@ -11,6 +11,39 @@ let _inMemoryCookies: Cookie[] | null = null; // Login lock to prevent concurrent login attempts let _loginLock: Promise | null = null; +// Cookie backup: preserved before clearCookieCache, restored on re-login failure +let _cookieBackup: Cookie[] | null = null; + +// Auth failure throttle: debounce consecutive re-login triggers from 500 errors +// Prevents thundering herd when server is slow and returns many 500s +let _authFailureCooldownUntil = 0; +const AUTH_FAILURE_COOLDOWN_MS = 15000; // 15s cooldown between re-login cycles + +/** + * Put all callers to wait during auth cooldown window. + * Returns true if auth is allowed (outside cooldown), false if throttled. + */ +export function tryAcquireAuthLock(): boolean { + const now = Date.now(); + if (now < _authFailureCooldownUntil) { + const remaining = _authFailureCooldownUntil - now; + logger.warn( + `Re-login throttled: ${Math.round(remaining / 1000)}s cooldown remaining. ` + + `Existing cookies are likely still valid — server 500 is a temporary slowdown.` + ); + return false; + } + return true; +} + +/** + * Called after a successful re-login to release the cooldown. + */ +export function releaseAuthCooldown(): void { + _authFailureCooldownUntil = Date.now() + AUTH_FAILURE_COOLDOWN_MS; + logger.info(`Auth cooldown set: ${AUTH_FAILURE_COOLDOWN_MS}ms to prevent thundering herd re-logins.`); +} + /** * Ensure only one login process runs at a time */ @@ -178,8 +211,40 @@ export async function saveCookiesToCache(cookies: Cookie[]): Promise { } } +/** + * Backup current cookies before clearing. Restored if re-login fails. + */ +export function backupCookies(): Cookie[] | null { + if (_inMemoryCookies) { + _cookieBackup = [..._inMemoryCookies]; + logger.info('Cookies backed up before clear.'); + } + return _cookieBackup; +} + +/** + * Restore cookies from backup after failed re-login. + */ +export async function restoreCookieBackup(): Promise { + if (_cookieBackup) { + _inMemoryCookies = _cookieBackup; + try { + await fs.promises.writeFile(COOKIE_FILE_PATH, JSON.stringify(_cookieBackup, null, 2), 'utf-8'); + logger.info('Cookies restored from backup successfully.'); + _cookieBackup = null; + return true; + } catch (error: any) { + logger.error('Failed to restore cookies from backup:', error.message); + return false; + } + } + logger.warn('No cookie backup available for restore.'); + return false; +} + /** * Clear cookie cache + * Prefer backupAndClearCookieCache() instead to preserve old cookies. */ export async function clearCookieCache(): Promise { _inMemoryCookies = null;