fix(scan): prevent progressive slowdown with mutex, batching, and connection pooling

- Add mutex to cron jobs to prevent overlapping runs
- Replace Promise.all with batched processing (50/batch) in updateStaleClubs
- Configure HTTP connection pooling with keep-alive (maxSockets: 50)
- Add memory monitoring to scan progress logs
- Reduce CONCURRENT_API_CALLS from 8 to 5 to reduce Sharp memory pressure

index.ts

@@ -2,6 +2,24 @@
 import express, { Request, Response } from 'express';
 import { config } from 'dotenv';
 import cors from 'cors';
+import http from 'http';
+import https from 'https';
+import axios from 'axios';
+
+// Configure HTTP connection pooling with keep-alive
+axios.defaults.httpAgent = new http.Agent({
+  keepAlive: true,
+  maxSockets: 50,
+  maxFreeSockets: 10,
+  timeout: 30000
+});
+
+axios.defaults.httpsAgent = new https.Agent({
+  keepAlive: true,
+  maxSockets: 50,
+  maxFreeSockets: 10,
+  timeout: 30000
+});
 import { fetchActivityData } from './engage-api/get-activity';
 import { structActivityData } from './engage-api/struct-activity';
 import { structStaffData } from './engage-api/struct-staff';
@@ -48,6 +66,10 @@ config();
 const USERNAME = process.env.API_USERNAME;
 const PASSWORD = process.env.API_PASSWORD;
 const PORT = process.env.PORT || 3000;
+
+// Mutex flags to prevent overlapping cron runs
+let isUpdatingClubs = false;
+let isUpdatingStaff = false;
 const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID;
 const allowedOriginsEnv = process.env.ALLOWED_ORIGINS || '*';
 const CLUB_CHECK_INTERVAL_SECONDS = parseInt(process.env.CLUB_CHECK_INTERVAL_SECONDS || '300', 10);
@@ -400,10 +422,28 @@ async function performBackgroundTasks(): Promise<void> {
     await cleanupOrphanedS3Images();
     
     logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`);
-    setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000);
+    setInterval(() => {
+      if (isUpdatingClubs) {
+        logger.warn('Previous club update still running, skipping this run');
+        return;
+      }
+      isUpdatingClubs = true;
+      updateStaleClubs().finally(() => {
+        isUpdatingClubs = false;
+      });
+    }, CLUB_CHECK_INTERVAL_SECONDS * 1000);
 
     logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`);
-    setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000);
+    setInterval(() => {
+      if (isUpdatingStaff) {
+        logger.warn('Previous staff update still running, skipping this run');
+        return;
+      }
+      isUpdatingStaff = true;
+      initializeOrUpdateStaffCache(false).finally(() => {
+        isUpdatingStaff = false;
+      });
+    }, STAFF_CHECK_INTERVAL_SECONDS * 1000);
     
     logger.info('Background initialization and periodic task setup complete.');
   } catch (error) {

services/cache-manager.ts

@@ -132,7 +132,8 @@ export async function initializeClubCache(): Promise<void> {
             successCount++;
             processedCount++;
             if (processedCount % 100 === 0) {
-              logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`);
+              const mem = process.memoryUsage();
+              logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount} | Heap: ${Math.round(mem.heapUsed/1024/1024)}MB`);
             }
           })
           .catch((error: unknown) => {
@@ -140,7 +141,8 @@ export async function initializeClubCache(): Promise<void> {
             processedCount++;
             logger.error(`Error processing activity ID ${activityId}:`, error);
             if (processedCount % 100 === 0) {
-              logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`);
+              const mem = process.memoryUsage();
+              logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount} | Heap: ${Math.round(mem.heapUsed/1024/1024)}MB`);
             }
           })
       )
@@ -183,7 +185,23 @@ export async function updateStaleClubs(): Promise<void> {
   }
 
   await cleanupOrphanedS3Images();
-  await Promise.all(promises);
+  
+  // Process promises in batches to prevent event loop blocking
+  const BATCH_SIZE = 50;
+  for (let i = 0; i < promises.length; i += BATCH_SIZE) {
+    const batch = promises.slice(i, i + BATCH_SIZE);
+    const batchNum = Math.floor(i / BATCH_SIZE) + 1;
+    const totalBatches = Math.ceil(promises.length / BATCH_SIZE);
+    
+    await Promise.all(batch);
+    logger.info(`Stale check batch ${batchNum}/${totalBatches} complete`);
+    
+    // Small delay between batches to prevent event loop blocking
+    if (i + BATCH_SIZE < promises.length) {
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+  }
+  
   logger.info('Stale club check finished.');
 }