diff --git a/engage-api/get-activity.ts b/engage-api/get-activity.ts index 715d720..cf4da55 100644 --- a/engage-api/get-activity.ts +++ b/engage-api/get-activity.ts @@ -54,7 +54,7 @@ async function testCookieValidityWithApi(cookieString: string): Promise logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`); const response = await axios.post(url, payload, { headers, - timeout: 20000 + timeout: 10000 }); // Check for 4xx errors (auth failures) @@ -112,8 +112,8 @@ async function getCompleteCookies(userName: string, userPwd: string): Promise { const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails'; const headers = { diff --git a/services/cache-manager.ts b/services/cache-manager.ts index 02bb7c7..f3cba1e 100644 --- a/services/cache-manager.ts +++ b/services/cache-manager.ts @@ -111,46 +111,57 @@ async function processSingleActivity(activityId: string): Promise { /** * Initialize the club cache by scanning through all activity IDs + * Processed in batches to prevent memory pressure from accumulating all promises upfront */ export async function initializeClubCache(): Promise { logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`); const totalIds = MAX_ACTIVITY_ID_SCAN - MIN_ACTIVITY_ID_SCAN + 1; + const BATCH_SIZE = 100; let processedCount = 0; let successCount = 0; let errorCount = 0; let skippedCount = 0; - const promises: Promise[] = []; - - for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) { - const activityId = String(i); - promises.push( - limit(() => - processSingleActivity(activityId) - .then(() => { - successCount++; - processedCount++; - if (processedCount % 100 === 0) { - const mem = process.memoryUsage(); - logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount} | Heap: ${Math.round(mem.heapUsed/1024/1024)}MB`); - } - }) - .catch((error: unknown) => { - errorCount++; - processedCount++; - logger.error(`Error processing activity ID ${activityId}:`, error); - if (processedCount % 100 === 0) { - const mem = process.memoryUsage(); - logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount} | Heap: ${Math.round(mem.heapUsed/1024/1024)}MB`); - } - }) - ) - ); - } + for (let batchStart = MIN_ACTIVITY_ID_SCAN; batchStart <= MAX_ACTIVITY_ID_SCAN; batchStart += BATCH_SIZE) { + const batchEnd = Math.min(batchStart + BATCH_SIZE - 1, MAX_ACTIVITY_ID_SCAN); + const batchPromises: Promise[] = []; + + logger.info(`Processing batch ${Math.floor(processedCount / BATCH_SIZE) + 1}/${Math.ceil(totalIds / BATCH_SIZE)} (IDs ${batchStart}-${batchEnd})`); + + for (let i = batchStart; i <= batchEnd; i++) { + const activityId = String(i); + batchPromises.push( + limit(() => + processSingleActivity(activityId) + .then(() => { + successCount++; + processedCount++; + if (processedCount % 100 === 0) { + const mem = process.memoryUsage(); + logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount} | Heap: ${Math.round(mem.heapUsed/1024/1024)}MB`); + } + }) + .catch((error: unknown) => { + errorCount++; + processedCount++; + logger.error(`Error processing activity ID ${activityId}:`, error); + if (processedCount % 100 === 0) { + const mem = process.memoryUsage(); + logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount} | Heap: ${Math.round(mem.heapUsed/1024/1024)}MB`); + } + }) + ) + ); + } - // Use allSettled to prevent single hung promise from blocking all - await Promise.allSettled(promises); + await Promise.allSettled(batchPromises); + batchPromises.length = 0; + + if (batchEnd < MAX_ACTIVITY_ID_SCAN) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } logger.info(`Initial club cache population finished.`); logger.info(`Summary: Total: ${totalIds}, Processed: ${processedCount}, Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`);