diff --git a/.gitignore b/.gitignore index 37a150c..db22bda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ node_modules -nkcs-engage.cookie.txt +cookies.json .env redis_data warp \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 459fe24..13e3161 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -18,6 +18,9 @@ services: context: . dockerfile: Dockerfile container_name: dsas-cca-backend + # Run as root to allow writing to volume-mounted cookies.json + # Alternative: Use named volume instead of bind mount + user: "0:0" ports: - "${PORT:-3000}:${PORT:-3000}" env_file: @@ -51,7 +54,8 @@ services: redis: image: "redis:8.0-alpine" container_name: dsas-cca-redis - command: redis-server --requirepass "dsas-cca" + # Add memory limits to prevent OOM + command: redis-server --requirepass "dsas-cca" --maxmemory 512mb --maxmemory-policy allkeys-lru volumes: - ./redis_data:/data restart: unless-stopped diff --git a/example.env b/example.env index 488b765..4505f42 100644 --- a/example.env +++ b/example.env @@ -18,6 +18,11 @@ STAFF_UPDATE_INTERVAL_MINS=360 CLUB_UPDATE_INTERVAL_MINS=360 LOG_LEVEL=info # Example: 'debug', 'info', 'warn', 'error' +# Cache TTL Configuration (in seconds) +ACTIVITY_CACHE_TTL=86400 # 24 hours for normal activity data +STAFF_CACHE_TTL=86400 # 24 hours for staff data +ERROR_CACHE_TTL=3600 # 1 hour for error states (allows retry) + # Proxy Configuration (Optional) # Set USE_PROXY=true to enable proxy for Playwright requests USE_PROXY=false diff --git a/services/cache-manager.ts b/services/cache-manager.ts index c7df654..7009132 100644 --- a/services/cache-manager.ts +++ b/services/cache-manager.ts @@ -96,24 +96,56 @@ async function processAndCacheActivity(activityId: string): Promise { logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`); + + const totalIds = MAX_ACTIVITY_ID_SCAN - MIN_ACTIVITY_ID_SCAN + 1; + let processedCount = 0; + let successCount = 0; + let errorCount = 0; + let skippedCount = 0; + const promises: Promise[] = []; for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) { const activityId = String(i); promises.push(limit(async () => { - const cachedData = await getActivityData(activityId); - if (!cachedData || - Object.keys(cachedData).length === 0 || - !cachedData.lastCheck || - cachedData.error) { - logger.debug(`Initializing cache for activity ID: ${activityId}`); - await processAndCacheActivity(activityId); + try { + const cachedData = await getActivityData(activityId); + + if (!cachedData || + Object.keys(cachedData).length === 0 || + !cachedData.lastCheck || + cachedData.error) { + + logger.debug(`Initializing cache for activity ID: ${activityId}`); + await processAndCacheActivity(activityId); + successCount++; + } else { + skippedCount++; + } + + processedCount++; + + // Log progress every 100 activities + if (processedCount % 100 === 0) { + logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`); + } + + } catch (error) { + errorCount++; + processedCount++; + logger.error(`Error processing activity ID ${activityId}:`, error); + + if (processedCount % 100 === 0) { + logger.info(`Progress: ${processedCount}/${totalIds} (${Math.round(processedCount/totalIds*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`); + } } })); } await Promise.all(promises); - logger.info('Initial club cache population finished.'); + + logger.info(`Initial club cache population finished.`); + logger.info(`Summary: Total: ${totalIds}, Processed: ${processedCount}, Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`); } /** diff --git a/services/redis-service.ts b/services/redis-service.ts index db52e1a..b7a17b4 100644 --- a/services/redis-service.ts +++ b/services/redis-service.ts @@ -8,6 +8,11 @@ config(); export const ACTIVITY_KEY_PREFIX = 'activity:'; // Exported for use in cache-manager const STAFF_KEY = 'staffs:all'; +// Cache TTL configuration (in seconds) +const ACTIVITY_CACHE_TTL = parseInt(process.env.ACTIVITY_CACHE_TTL || '86400', 10); // Default: 24 hours +const STAFF_CACHE_TTL = parseInt(process.env.STAFF_CACHE_TTL || '86400', 10); // Default: 24 hours +const ERROR_CACHE_TTL = parseInt(process.env.ERROR_CACHE_TTL || '3600', 10); // Default: 1 hour for errors + // Always create a new client instance with .env config const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379'; let redisClient: RedisClient | null = null; @@ -39,17 +44,25 @@ export async function getActivityData(activityId: string): Promise { } /** - * Sets activity data in Redis. + * Sets activity data in Redis with TTL. * @param activityId - The activity ID to set * @param data - The activity data object + * @param ttl - Optional TTL in seconds (defaults to ACTIVITY_CACHE_TTL, or ERROR_CACHE_TTL if data has error) */ -export async function setActivityData(activityId: string, data: any): Promise { +export async function setActivityData(activityId: string, data: any, ttl?: number): Promise { if (!redisClient) { logger.warn('Redis client not available, skipping setActivityData'); return; } try { - await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data)); + // Use shorter TTL for error states to allow retry + const expiration = data?.error ? ERROR_CACHE_TTL : (ttl || ACTIVITY_CACHE_TTL); + // Bun's RedisClient doesn't have setEx, use raw SETEX command + await redisClient.send('SETEX', [ + `${ACTIVITY_KEY_PREFIX}${activityId}`, + String(expiration), + JSON.stringify(data) + ]); } catch (err) { logger.error(`Error setting activity ${activityId} in Redis:`, err); } @@ -74,16 +87,23 @@ export async function getStaffData(): Promise { } /** - * Sets staff data in Redis. + * Sets staff data in Redis with TTL. * @param data - The staff data object + * @param ttl - Optional TTL in seconds (defaults to STAFF_CACHE_TTL) */ -export async function setStaffData(data: any): Promise { +export async function setStaffData(data: any, ttl?: number): Promise { if (!redisClient) { logger.warn('Redis client not available, skipping setStaffData'); return; } try { - await redisClient.set(STAFF_KEY, JSON.stringify(data)); + const expiration = ttl || STAFF_CACHE_TTL; + // Use raw SETEX command for TTL support + await redisClient.send('SETEX', [ + STAFF_KEY, + String(expiration), + JSON.stringify(data) + ]); } catch (err) { logger.error('Error setting staff data in Redis:', err); } @@ -103,8 +123,11 @@ export async function getAllActivityKeys(): Promise { // Using raw SCAN command since Bun's RedisClient doesn't have a scan method const keys: string[] = []; let cursor = '0'; + let iteration = 0; + const MAX_ITERATIONS = 1000; // Safety limit to prevent infinite loops do { + iteration++; // Use send method to execute raw Redis commands const result = await redisClient.send('SCAN', [ cursor, @@ -114,15 +137,24 @@ export async function getAllActivityKeys(): Promise { '100' ]); - cursor = result[0]; + // Force convert to string to ensure type consistency (Bun may return Buffer) + cursor = String(result[0] ?? '0'); const foundKeys = result[1] || []; + logger.debug(`SCAN iteration ${iteration}: cursor=${cursor}, found ${foundKeys.length} keys, total=${keys.length + foundKeys.length}`); + // Add the found keys to our array keys.push(...foundKeys); + // Prevent infinite loop + if (iteration >= MAX_ITERATIONS) { + logger.warn(`SCAN reached max iterations (${MAX_ITERATIONS}). May have incomplete results.`); + break; + } + } while (cursor !== '0'); - logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`); + logger.info(`Found ${keys.length} activity keys in Redis after ${iteration} SCAN iterations.`); return keys; } catch (err) { logger.error('Error getting all activity keys from Redis using SCAN:', err);