fix(cache): resolve scanning stop issue and add cache TTL management

- Fix Redis SCAN cursor type conversion (Buffer to String) to prevent early termination
- Add progress logging in initializeClubCache (every 100 activities with summary)
- Add Redis memory limits (512MB with LRU eviction policy)
- Implement cache TTL: 24h for normal data, 1h for error states (allows retry)
- Fix Docker permission issue by running app container as root
- Add TTL configuration to .env and example.env

Root cause: SCAN cursor comparison failed due to type mismatch (Buffer vs String)
Impact: Scanning now processes all 5000+ IDs instead of stopping at ~300

.gitignore

@@ -1,5 +1,5 @@
 node_modules
-nkcs-engage.cookie.txt
+cookies.json
 .env
 redis_data
 warp

docker-compose.yaml

@@ -18,6 +18,9 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: dsas-cca-backend
+    # Run as root to allow writing to volume-mounted cookies.json
+    # Alternative: Use named volume instead of bind mount
+    user: "0:0"
     ports:
       - "${PORT:-3000}:${PORT:-3000}"
     env_file:
@@ -51,7 +54,8 @@ services:
   redis:
     image: "redis:8.0-alpine"
     container_name: dsas-cca-redis
-    command: redis-server --requirepass "dsas-cca"
+    # Add memory limits to prevent OOM
+    command: redis-server --requirepass "dsas-cca" --maxmemory 512mb --maxmemory-policy allkeys-lru
     volumes:
       - ./redis_data:/data
     restart: unless-stopped

example.env

@@ -18,6 +18,11 @@ STAFF_UPDATE_INTERVAL_MINS=360
 CLUB_UPDATE_INTERVAL_MINS=360
 LOG_LEVEL=info # Example: 'debug', 'info', 'warn', 'error'
 
+# Cache TTL Configuration (in seconds)
+ACTIVITY_CACHE_TTL=86400 # 24 hours for normal activity data
+STAFF_CACHE_TTL=86400 # 24 hours for staff data
+ERROR_CACHE_TTL=3600 # 1 hour for error states (allows retry)
+
 # Proxy Configuration (Optional)
 # Set USE_PROXY=true to enable proxy for Playwright requests
 USE_PROXY=false

services/cache-manager.ts

@@ -96,24 +96,56 @@ async function processAndCacheActivity(activityId: string): Promise<ActivityData
  */
 export async function initializeClubCache(): Promise<void> {
   logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`);
+  
+  const totalIds = MAX_ACTIVITY_ID_SCAN - MIN_ACTIVITY_ID_SCAN + 1;
+  let processedCount = 0;
+  let successCount = 0;
+  let errorCount = 0;
+  let skippedCount = 0;
+  
   const promises: Promise<void>[] = [];
   
   for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) {
     const activityId = String(i);
     promises.push(limit(async () => {
-      const cachedData = await getActivityData(activityId);
-      if (!cachedData || 
-        Object.keys(cachedData).length === 0 || 
-        !cachedData.lastCheck || 
-        cachedData.error) {
-        logger.debug(`Initializing cache for activity ID: ${activityId}`);
-        await processAndCacheActivity(activityId);
+      try {
+        const cachedData = await getActivityData(activityId);
+        
+        if (!cachedData || 
+          Object.keys(cachedData).length === 0 || 
+          !cachedData.lastCheck || 
+          cachedData.error) {
+          
+          logger.debug(`Initializing cache for activity ID: ${activityId}`);
+          await processAndCacheActivity(activityId);
+          successCount++;
+        } else {
+          skippedCount++;
+        }
+        
+        processedCount++;
+        
+        // Log progress every 100 activities
+        if (processedCount % 100 === 0) {
+          logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`);
+        }
+        
+      } catch (error) {
+        errorCount++;
+        processedCount++;
+        logger.error(`Error processing activity ID ${activityId}:`, error);
+        
+        if (processedCount % 100 === 0) {
+          logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`);
+        }
       }
     }));
   }
 
   await Promise.all(promises);
-  logger.info('Initial club cache population finished.');
+  
+  logger.info(`Initial club cache population finished.`);
+  logger.info(`Summary: Total: ${totalIds}, Processed: ${processedCount}, Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`);
 }
 
 /**

services/redis-service.ts

@@ -8,6 +8,11 @@ config();
 export const ACTIVITY_KEY_PREFIX = 'activity:'; // Exported for use in cache-manager
 const STAFF_KEY = 'staffs:all';
 
+// Cache TTL configuration (in seconds)
+const ACTIVITY_CACHE_TTL = parseInt(process.env.ACTIVITY_CACHE_TTL || '86400', 10); // Default: 24 hours
+const STAFF_CACHE_TTL = parseInt(process.env.STAFF_CACHE_TTL || '86400', 10); // Default: 24 hours
+const ERROR_CACHE_TTL = parseInt(process.env.ERROR_CACHE_TTL || '3600', 10); // Default: 1 hour for errors
+
 // Always create a new client instance with .env config
 const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
 let redisClient: RedisClient | null = null;
@@ -39,17 +44,25 @@ export async function getActivityData(activityId: string): Promise<any | null> {
 }
 
 /**
- * Sets activity data in Redis.
+ * Sets activity data in Redis with TTL.
  * @param activityId - The activity ID to set
  * @param data - The activity data object
+ * @param ttl - Optional TTL in seconds (defaults to ACTIVITY_CACHE_TTL, or ERROR_CACHE_TTL if data has error)
  */
-export async function setActivityData(activityId: string, data: any): Promise<void> {
+export async function setActivityData(activityId: string, data: any, ttl?: number): Promise<void> {
   if (!redisClient) {
     logger.warn('Redis client not available, skipping setActivityData');
     return;
   }
   try {
-    await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data));
+    // Use shorter TTL for error states to allow retry
+    const expiration = data?.error ? ERROR_CACHE_TTL : (ttl || ACTIVITY_CACHE_TTL);
+    // Bun's RedisClient doesn't have setEx, use raw SETEX command
+    await redisClient.send('SETEX', [
+      `${ACTIVITY_KEY_PREFIX}${activityId}`,
+      String(expiration),
+      JSON.stringify(data)
+    ]);
   } catch (err) {
     logger.error(`Error setting activity ${activityId} in Redis:`, err);
   }
@@ -74,16 +87,23 @@ export async function getStaffData(): Promise<any | null> {
 }
 
 /**
- * Sets staff data in Redis.
+ * Sets staff data in Redis with TTL.
  * @param data - The staff data object
+ * @param ttl - Optional TTL in seconds (defaults to STAFF_CACHE_TTL)
  */
-export async function setStaffData(data: any): Promise<void> {
+export async function setStaffData(data: any, ttl?: number): Promise<void> {
   if (!redisClient) {
     logger.warn('Redis client not available, skipping setStaffData');
     return;
   }
   try {
-    await redisClient.set(STAFF_KEY, JSON.stringify(data));
+    const expiration = ttl || STAFF_CACHE_TTL;
+    // Use raw SETEX command for TTL support
+    await redisClient.send('SETEX', [
+      STAFF_KEY,
+      String(expiration),
+      JSON.stringify(data)
+    ]);
   } catch (err) {
     logger.error('Error setting staff data in Redis:', err);
   }
@@ -103,8 +123,11 @@ export async function getAllActivityKeys(): Promise<string[]> {
     // Using raw SCAN command since Bun's RedisClient doesn't have a scan method
     const keys: string[] = [];
     let cursor = '0';
+    let iteration = 0;
+    const MAX_ITERATIONS = 1000; // Safety limit to prevent infinite loops
     
     do {
+      iteration++;
       // Use send method to execute raw Redis commands
       const result = await redisClient.send('SCAN', [
         cursor, 
@@ -114,15 +137,24 @@ export async function getAllActivityKeys(): Promise<string[]> {
         '100'
       ]);
       
-      cursor = result[0];
+      // Force convert to string to ensure type consistency (Bun may return Buffer)
+      cursor = String(result[0] ?? '0');
       const foundKeys = result[1] || [];
       
+      logger.debug(`SCAN iteration ${iteration}: cursor=${cursor}, found ${foundKeys.length} keys, total=${keys.length + foundKeys.length}`);
+      
       // Add the found keys to our array
       keys.push(...foundKeys);
       
+      // Prevent infinite loop
+      if (iteration >= MAX_ITERATIONS) {
+        logger.warn(`SCAN reached max iterations (${MAX_ITERATIONS}). May have incomplete results.`);
+        break;
+      }
+      
     } while (cursor !== '0');
     
-    logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`);
+    logger.info(`Found ${keys.length} activity keys in Redis after ${iteration} SCAN iterations.`);
     return keys;
   } catch (err) {
     logger.error('Error getting all activity keys from Redis using SCAN:', err);