fix(auth): prevent cookie loss during remote server timeout storms
Server timeouts caused orphaned fetchActivityData calls to fire clearCookieCache() asynchronously, destroying cookies for all concurrent callers. Three fixes: 1. Replace Promise.race timeout with AbortController to properly cancel orphaned fetches and prevent delayed clearCookieCache() calls 2. Add cookie backup/restore — backupCookies() before clearCookieCache(), restoreCookieBackup() if re-login fails, so cookies are never lost 3. Add 15s auth failure throttle to block thundering herd re-logins when server slowdowns generate many 500 errors simultaneously
This commit is contained in:
@@ -52,14 +52,33 @@ async function processAndCacheActivity(activityId: string, forceUpdate: boolean
|
||||
throw new Error('API username or password not configured');
|
||||
}
|
||||
|
||||
// Add timeout protection for the entire fetch operation
|
||||
// Add timeout protection via AbortController - properly cancels orphaned fetches
|
||||
logger.debug(`Fetching activity data for ID: ${activityId}`);
|
||||
const activityJson = await Promise.race([
|
||||
fetchActivityData(activityId, USERNAME, PASSWORD, false),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`Timeout fetching activity ${activityId} after ${CRAWLER_REQUEST_TIMEOUT_MS}ms`)), CRAWLER_REQUEST_TIMEOUT_MS + 5000)
|
||||
)
|
||||
]);
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(
|
||||
() => controller.abort(),
|
||||
CRAWLER_REQUEST_TIMEOUT_MS + 5000
|
||||
);
|
||||
|
||||
let activityJson: any = null;
|
||||
try {
|
||||
activityJson = await fetchActivityData(
|
||||
activityId,
|
||||
USERNAME,
|
||||
PASSWORD,
|
||||
false,
|
||||
controller.signal
|
||||
);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
|
||||
if (controller.signal.aborted) {
|
||||
logger.warn(`Request for activity ${activityId} timed out after ${CRAWLER_REQUEST_TIMEOUT_MS + 5000}ms. Cancelling orphaned fetch.`);
|
||||
// Preserve existing cache on timeout
|
||||
const existingData = await getActivityData(activityId);
|
||||
return existingData || { lastCheck: new Date().toISOString(), error: `Timeout after ${CRAWLER_REQUEST_TIMEOUT_MS + 5000}ms` };
|
||||
}
|
||||
let structuredActivity: ActivityData;
|
||||
|
||||
if (!activityJson) {
|
||||
|
||||
@@ -11,6 +11,39 @@ let _inMemoryCookies: Cookie[] | null = null;
|
||||
// Login lock to prevent concurrent login attempts
|
||||
let _loginLock: Promise<Cookie[]> | null = null;
|
||||
|
||||
// Cookie backup: preserved before clearCookieCache, restored on re-login failure
|
||||
let _cookieBackup: Cookie[] | null = null;
|
||||
|
||||
// Auth failure throttle: debounce consecutive re-login triggers from 500 errors
|
||||
// Prevents thundering herd when server is slow and returns many 500s
|
||||
let _authFailureCooldownUntil = 0;
|
||||
const AUTH_FAILURE_COOLDOWN_MS = 15000; // 15s cooldown between re-login cycles
|
||||
|
||||
/**
|
||||
* Put all callers to wait during auth cooldown window.
|
||||
* Returns true if auth is allowed (outside cooldown), false if throttled.
|
||||
*/
|
||||
export function tryAcquireAuthLock(): boolean {
|
||||
const now = Date.now();
|
||||
if (now < _authFailureCooldownUntil) {
|
||||
const remaining = _authFailureCooldownUntil - now;
|
||||
logger.warn(
|
||||
`Re-login throttled: ${Math.round(remaining / 1000)}s cooldown remaining. ` +
|
||||
`Existing cookies are likely still valid — server 500 is a temporary slowdown.`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Called after a successful re-login to release the cooldown.
|
||||
*/
|
||||
export function releaseAuthCooldown(): void {
|
||||
_authFailureCooldownUntil = Date.now() + AUTH_FAILURE_COOLDOWN_MS;
|
||||
logger.info(`Auth cooldown set: ${AUTH_FAILURE_COOLDOWN_MS}ms to prevent thundering herd re-logins.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure only one login process runs at a time
|
||||
*/
|
||||
@@ -178,8 +211,40 @@ export async function saveCookiesToCache(cookies: Cookie[]): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Backup current cookies before clearing. Restored if re-login fails.
|
||||
*/
|
||||
export function backupCookies(): Cookie[] | null {
|
||||
if (_inMemoryCookies) {
|
||||
_cookieBackup = [..._inMemoryCookies];
|
||||
logger.info('Cookies backed up before clear.');
|
||||
}
|
||||
return _cookieBackup;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore cookies from backup after failed re-login.
|
||||
*/
|
||||
export async function restoreCookieBackup(): Promise<boolean> {
|
||||
if (_cookieBackup) {
|
||||
_inMemoryCookies = _cookieBackup;
|
||||
try {
|
||||
await fs.promises.writeFile(COOKIE_FILE_PATH, JSON.stringify(_cookieBackup, null, 2), 'utf-8');
|
||||
logger.info('Cookies restored from backup successfully.');
|
||||
_cookieBackup = null;
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
logger.error('Failed to restore cookies from backup:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
logger.warn('No cookie backup available for restore.');
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear cookie cache
|
||||
* Prefer backupAndClearCookieCache() instead to preserve old cookies.
|
||||
*/
|
||||
export async function clearCookieCache(): Promise<void> {
|
||||
_inMemoryCookies = null;
|
||||
|
||||
Reference in New Issue
Block a user