fix(auth): prevent cookie loss during remote server timeout storms

Server timeouts caused orphaned fetchActivityData calls to fire clearCookieCache()
asynchronously, destroying cookies for all concurrent callers. Three fixes:

1. Replace Promise.race timeout with AbortController to properly cancel
   orphaned fetches and prevent delayed clearCookieCache() calls
2. Add cookie backup/restore — backupCookies() before clearCookieCache(),
   restoreCookieBackup() if re-login fails, so cookies are never lost
3. Add 15s auth failure throttle to block thundering herd re-logins when
   server slowdowns generate many 500 errors simultaneously

.gitignore

@@ -1,4 +1,5 @@
 node_modules
-nkcs-engage.cookie.txt
+cookies.json
 .env
-redis_data
+redis_data
+warp

Dockerfile

@@ -1,15 +1,53 @@
 FROM oven/bun:latest
 
 ENV NODE_ENV=production
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+ENV DEBIAN_FRONTEND=noninteractive
 
 WORKDIR /usr/src/app
 
+# Install Playwright system dependencies
+RUN apt-get update && apt-get install -y \
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libxkbcommon0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libasound2 \
+    libpango-1.0-0 \
+    libcairo2 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy dependency files
 COPY package.json bun.lock ./
 
+# Install dependencies (including Playwright)
 RUN bun install --production
 
+# Install Playwright browsers
+RUN bunx playwright install chromium --with-deps || true
+
+# Copy application code
 COPY . .
 
+# Make startup script executable
+RUN chmod +x startup.sh
+
+# Create non-root user for security
+RUN adduser --disabled-password --gecos '' appuser && \
+    chown -R appuser:appuser /usr/src/app
+
+USER appuser
+
 EXPOSE 3000
 
-CMD ["bun", "start"]
+# Use startup script
+CMD ["/bin/sh", "startup.sh"]

bun.lock

@@ -1,24 +1,26 @@
 {
   "lockfileVersion": 1,
+  "configVersion": 0,
   "workspaces": {
     "": {
       "name": "dsas-cca-backend-bun",
       "dependencies": {
+        "@playwright/test": "^1.49.0",
         "axios": "^1.9.0",
         "cors": "^2.8.5",
         "crypto": "^1.0.1",
         "dotenv": "^16.5.0",
         "express": "^5.1.0",
-        "p-limit": "^6.2.0",
         "pangu": "^4.0.7",
         "sharp": "^0.34.1",
         "uuid": "^11.1.0",
       },
       "devDependencies": {
         "@types/bun": "latest",
+        "typescript-language-server": "^5.1.3",
       },
       "peerDependencies": {
-        "typescript": "^5",
+        "typescript": "^5.9.3",
       },
     },
   },
@@ -65,6 +67,8 @@
 
     "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.1", "", { "os": "win32", "cpu": "x64" }, "sha512-hw1iIAHpNE8q3uMIRCgGOeDoz9KtFNarFLQclLxr/LK1VBkj8nby18RjFvr6aP7USRYAjTZW6yisnBWMX571Tw=="],
 
+    "@playwright/test": ["@playwright/test@1.59.1", "", { "dependencies": { "playwright": "1.59.1" }, "bin": { "playwright": "cli.js" } }, "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg=="],
+
     "@types/bun": ["@types/bun@1.2.13", "", { "dependencies": { "bun-types": "1.2.13" } }, "sha512-u6vXep/i9VBxoJl3GjZsl/BFIsvML8DfVDO0RYLEwtSZSp981kEO1V5NwRcO1CPJ7AmvpbnDCiMKo3JvbDEjAg=="],
 
     "@types/node": ["@types/node@22.15.17", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-wIX2aSZL5FE+MR0JlvF87BNVrtFWf6AE6rxSE9X7OwnVvoyCQjpzSRJ+M87se/4QCkCiebQAqrJ0y6fwIyi7nw=="],
@@ -147,6 +151,8 @@
 
     "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
 
+    "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="],
+
     "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
 
     "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
@@ -195,14 +201,16 @@
 
     "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
 
-    "p-limit": ["p-limit@6.2.0", "", { "dependencies": { "yocto-queue": "^1.1.1" } }, "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA=="],
-
     "pangu": ["pangu@4.0.7", "", { "bin": { "pangu": "./dist/node/cli.js" } }, "sha512-weZKJIwwy5gjt4STGVUH9bix3BGk7wZ2ahtIypwe3e/mllsrIZIvtfLx1dPX56GcpZFOCFKmeqI1qVuB9enRzA=="],
 
     "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
 
     "path-to-regexp": ["path-to-regexp@8.2.0", "", {}, "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ=="],
 
+    "playwright": ["playwright@1.59.1", "", { "dependencies": { "playwright-core": "1.59.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw=="],
+
+    "playwright-core": ["playwright-core@1.59.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg=="],
+
     "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="],
 
     "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="],
@@ -247,7 +255,9 @@
 
     "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
 
-    "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
+    "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
+
+    "typescript-language-server": ["typescript-language-server@5.1.3", "", { "bin": { "typescript-language-server": "lib/cli.mjs" } }, "sha512-r+pAcYtWdN8tKlYZPwiiHNA2QPjXnI02NrW5Sf2cVM3TRtuQ3V9EKKwOxqwaQ0krsaEXk/CbN90I5erBuf84Vg=="],
 
     "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
 
@@ -259,8 +269,6 @@
 
     "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
 
-    "yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="],
-
     "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
 
     "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],

docker-compose.yaml

@@ -4,15 +4,25 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: dsas-cca-backend
+    # Run as root to allow writing to volume-mounted cookies.json
+    # Alternative: Use named volume instead of bind mount
+    user: "0:0"
     ports:
-      - "${PORT}:${PORT}"
+      - "${PORT:-3000}:${PORT:-3000}"
     env_file:
       - .env
+    environment:
+      - NODE_ENV=production
+      - PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
     restart: unless-stopped
     depends_on:
-      - redis
+      redis:
-    networks:
+        condition: service_healthy
-      - cca_network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "15m"
+        max-file: "3"
 
   redis:
     image: "redis:8.0-alpine"
@@ -21,14 +31,13 @@ services:
     volumes:
       - ./redis_data:/data
     restart: unless-stopped
-    networks:
-      - cca_network
     healthcheck:
       test: ["CMD", "redis-cli", "-a", "dsas-cca", "ping"]
       interval: 10s
       timeout: 5s
       retries: 5
-
+    logging:
-networks:
+      driver: "json-file"
-  cca_network:
+      options:
-    driver: bridge
+        max-size: "15m"
+        max-file: "3"

engage-api/get-activity.ts

@@ -1,13 +1,21 @@
 // engage-api/get-activity.ts
-import axios from 'axios';
+import axios, { type AxiosRequestConfig } from 'axios';
-import { readFile,writeFile,unlink } from 'fs/promises';
-import { resolve } from 'path';
 import { logger } from '../utils/logger';
+import {
+  ensureSingleLogin,
+  saveCookiesToCache,
+  clearCookieCache,
+  getCachedCookieString,
+  backupCookies,
+  restoreCookieBackup,
+  tryAcquireAuthLock,
+  releaseAuthCooldown
+} from '../services/playwright-auth';
 
 // Define interfaces for our data structures
 interface ActivityResponse {
   d: string;
-  isError ? : boolean;
+  isError?: boolean;
   [key: string]: any;
 }
 
@@ -15,206 +23,40 @@ interface ActivityResponse {
 class AuthenticationError extends Error {
   status: number;
 
-  constructor(message: string = "Authentication failed, cookie may be invalid.", status ? : number) {
+  constructor(message: string = "Authentication failed, cookie may be invalid.", status?: number) {
     super(message);
     this.name = "AuthenticationError";
     this.status = status || 0;
   }
 }
 
-// In Bun, we can use import.meta.dir instead of the Node.js __dirname approach
+/**
-const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt');
+ * Get complete cookies using Playwright with single login lock
-let _inMemoryCookie: string | null = null;
+ */
+async function getCompleteCookies(userName: string, userPwd: string): Promise<string> {
+  logger.info('Attempting to get complete cookie string using Playwright login...');
+  
+  const cookies = await ensureSingleLogin(userName, userPwd);
+  
+  if (!cookies || cookies.length === 0) {
+    throw new Error("Login failed: Could not obtain cookies.");
+  }
 
-// Cookie Cache Helper Functions
+  const cookieString = cookies.map((c: any) => `${c.name}=${c.value}`).join('; ');
-async function loadCachedCookie(): Promise < string | null > {
+  return cookieString;
-  if (_inMemoryCookie) {
-    logger.debug("Using in-memory cached cookie.");
-    return _inMemoryCookie;
-  }
-  try {
-    const cookieFromFile = await readFile(COOKIE_FILE_PATH, 'utf8');
-    if (cookieFromFile) {
-      _inMemoryCookie = cookieFromFile;
-      logger.debug("Loaded cookie from file cache.");
-      return _inMemoryCookie;
-    }
-  } catch (err: any) {
-    if (err.code === 'ENOENT') {
-      logger.debug("Cookie cache file not found. No cached cookie loaded.");
-    } else {
-      logger.warn("Error loading cookie from file:", err.message);
-    }
-  }
-  return null;
-}
-
-async function saveCookieToCache(cookieString: string): Promise < void > {
-  if (!cookieString) {
-    logger.warn("Attempted to save an empty or null cookie. Aborting save.");
-    return;
-  }
-  _inMemoryCookie = cookieString;
-  try {
-    await writeFile(COOKIE_FILE_PATH, cookieString, 'utf8');
-    logger.debug("Cookie saved to file cache.");
-  } catch (err: any) {
-    logger.error("Error saving cookie to file:", err.message);
-  }
-}
-
-async function clearCookieCache(): Promise < void > {
-  _inMemoryCookie = null;
-  try {
-    await unlink(COOKIE_FILE_PATH);
-    logger.debug("Cookie cache file deleted.");
-  } catch (err: any) {
-    if (err.code !== 'ENOENT') {
-      logger.error("Error deleting cookie file:", err.message);
-    } else {
-      logger.debug("Cookie cache file did not exist, no need to delete.");
-    }
-  }
-}
-
-async function testCookieValidity(cookieString: string): Promise < boolean > {
-  if (!cookieString) return false;
-  logger.debug("Testing cookie validity...");
-
-  const MAX_RETRIES = 3;
-  let attempt = 0;
-
-  while (attempt < MAX_RETRIES) {
-    try {
-      attempt++;
-      const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
-      const headers = {
-        'Content-Type': 'application/json; charset=UTF-8',
-        'Cookie': cookieString,
-        'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
-      };
-      const payload = {
-        "activityID": "3350"
-      };
-
-      logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`);
-      await axios.post(url, payload, {
-        headers,
-        timeout: 20000
-      });
-
-      logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
-      return true;
-    } catch (error: any) {
-      logger.warn(`Cookie validity test failed (attempt ${attempt}/${MAX_RETRIES}).`);
-      if (error.response) {
-        logger.warn(`Cookie test API response status: ${error.response.status}.`);
-      } else {
-        logger.warn(`Network/other error: ${error.message}`);
-      }
-
-      if (attempt >= MAX_RETRIES) {
-        logger.warn("Max retries reached. Cookie is likely invalid or expired.");
-        return false;
-      }
-    }
-  }
-  return false;
-}
-
-// Core API Interaction Functions
-async function getSessionId(): Promise < string | null > {
-  const url = 'https://engage.nkcswx.cn/Login.aspx';
-  try {
-    const response = await axios.get(url, {
-      headers: {
-        'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)'
-      }
-    });
-    const setCookieHeader = response.headers['set-cookie'];
-    if (setCookieHeader && setCookieHeader.length > 0) {
-      const sessionIdCookie = setCookieHeader.find(cookie => cookie.trim().startsWith('ASP.NET_SessionId='));
-      if (sessionIdCookie) {
-        logger.debug('ASP.NET_SessionId created');
-        return sessionIdCookie.split(';')[0] || null; // Ensure a fallback to `null` if splitting fails
-      }
-      return null; // Explicitly return `null` if no cookie is found
-    }
-    logger.error("No ASP.NET_SessionId cookie found in Set-Cookie header.");
-    return null;
-  } catch (error: any) {
-    logger.error(`Error in getSessionId: ${error.response ? `${error.response.status} - ${error.response.statusText}` : error.message}`);
-    throw error;
-  }
-}
-
-async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise < string | null > {
-  const url = 'https://engage.nkcswx.cn/Login.aspx';
-  try {
-    let templateData = await readFile(templateFilePath, 'utf8');
-    const postData = templateData
-      .replace('{{USERNAME}}', userName)
-      .replace('{{PASSWORD}}', userPwd);
-    const headers = {
-      'Content-Type': 'application/x-www-form-urlencoded',
-      'Cookie': sessionId,
-      'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
-      'Referer': 'https://engage.nkcswx.cn/Login.aspx'
-    };
-    logger.debug('Getting .ASPXFORMSAUTH');
-    const response = await axios.post(url, postData, {
-      headers,
-      maxRedirects: 0,
-      validateStatus: (status) => status >= 200 && status < 400
-    });
-    const setCookieHeader = response.headers['set-cookie'];
-    let formsAuthCookieValue = null;
-    if (setCookieHeader && setCookieHeader.length > 0) {
-      const aspxAuthCookies = setCookieHeader.filter(cookie => cookie.trim().startsWith('.ASPXFORMSAUTH='));
-      if (aspxAuthCookies.length > 0) {
-        for (let i = aspxAuthCookies.length - 1; i >= 0; i--) {
-          const cookieCandidateParts = aspxAuthCookies[i].split(';');
-          if (cookieCandidateParts.length > 0 && cookieCandidateParts[0] !== undefined) { // Explicit check
-            const firstPart = cookieCandidateParts[0].trim();
-            if (firstPart.length > '.ASPXFORMSAUTH='.length && firstPart.substring('.ASPXFORMSAUTH='.length).length > 0) {
-              formsAuthCookieValue = firstPart;
-              break;
-            }
-          }
-        }
-      }
-    }
-    if (formsAuthCookieValue) {
-      logger.debug('.ASPXFORMSAUTH cookie obtained.');
-      return formsAuthCookieValue;
-    } else {
-      logger.error("No valid .ASPXFORMSAUTH cookie found. Headers:", setCookieHeader || "none");
-      return null;
-    }
-  } catch (error: any) {
-    if (error.code === 'ENOENT') logger.error(`Error: Template file '${templateFilePath}' not found.`);
-    else logger.error(`Error in getMSAUTH: ${error.message}`);
-    throw error;
-  }
-}
-
-async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise < string > {
-  logger.debug('Attempting to get complete cookie string (login process).');
-  const sessionId = await getSessionId();
-  if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId.");
-
-  const msAuth = await getMSAUTH(sessionId, userName, userPwd, templateFilePath);
-  if (!msAuth) throw new Error("Login failed: Could not obtain .ASPXFORMSAUTH cookie.");
-
-  return `${sessionId}; ${msAuth}`;
 }
 
+/**
+ * Get activity details from API
+ * Only returns data on HTTP 200. Returns null on any error (5xx, timeout, etc.)
+ */
 async function getActivityDetailsRaw(
   activityId: string,
   cookies: string,
   maxRetries: number = 3,
-  timeoutMilliseconds: number = 20000
+  timeoutMilliseconds: number = 10000,
-): Promise < string | null > {
+  signal?: AbortSignal
+): Promise<string | null> {
   const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
   const headers = {
     'Content-Type': 'application/json; charset=UTF-8',
@@ -227,12 +69,41 @@ async function getActivityDetailsRaw(
   };
 
   for (let attempt = 0; attempt < maxRetries; attempt++) {
+    if (signal?.aborted) {
+      logger.debug(`Activity ${activityId} aborted before attempt ${attempt + 1}`);
+      return null;
+    }
     try {
+      logger.debug(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} - Sending POST request to ${url}`);
       const response = await axios.post(url, payload, {
         headers,
         timeout: timeoutMilliseconds,
-        responseType: 'text'
+        responseType: 'text',
+        signal,
+        maxRedirects: 5
       });
+      
+      // CRITICAL: Only accept HTTP 200. Reject all other status codes including 5xx
+      if (response.status !== 200) {
+        logger.error(`Non-200 status ${response.status} for activity ${activityId}. NOT updating cache to preserve local data.`);
+        
+        // IMPORTANT: Only 500 is cookie expiration. Other 5xx (502/503/504) are real server outages.
+        // The backend returns 500 when cookie is expired but session not yet invalidated.
+        // It takes several hours before it returns 401/403.
+        // 502/503/504 are real server errors (bad gateway, service unavailable, gateway timeout)
+        if (response.status === 500) {
+          logger.warn(`Server error 500 - this is cookie expiration. Throwing AuthenticationError to trigger immediate re-login.`);
+          throw new AuthenticationError(`Received 500 for activity ${activityId} - expired cookie`, 500);
+        } else if (response.status >= 500 && response.status < 600) {
+          // Real server outage (502/503/504), preserve cache and don't re-login
+          logger.error(`Real server outage ${response.status} - preserving local cache, not re-login.`);
+        }
+        
+        // Return null immediately on non-200 errors
+        return null;
+      }
+      
+      logger.debug(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} - Received response status ${response.status}`);
       const outerData = JSON.parse(response.data);
       if (outerData && typeof outerData.d === 'string') {
         const innerData = JSON.parse(outerData.d);
@@ -245,8 +116,11 @@ async function getActivityDetailsRaw(
         logger.error(`Unexpected API response structure for activity ${activityId}.`);
       }
     } catch (error: any) {
-      // Check if response status is in 4xx range (400-499) to trigger auth error
+      
-      if (error.response && error.response.status >= 400 && error.response.status < 500) {
+      // Only treat 401 (Unauthorized) and 403 (Forbidden) as authentication errors
+      // 404 (Not Found) is valid - activity doesn't exist
+      // Other 4xx/5xx errors should not trigger re-authentication
+      if (error.response && (error.response.status === 401 || error.response.status === 403)) {
         logger.warn(`Authentication error (${error.response.status}) while fetching activity ${activityId}. Cookie may be invalid.`);
         throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
       }
@@ -254,10 +128,21 @@ async function getActivityDetailsRaw(
 
       if (error.response) {
         logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
+        // IMPORTANT: Only 500 is cookie expiration. Other 5xx (502/503/504) are real server outages.
+        // The backend returns 500 when cookie is expired but session not yet invalidated.
+        // 502/503/504 are real server errors (bad gateway, service unavailable, gateway timeout)
+        if (error.response.status === 500) {
+          logger.warn(`Server error 500 - this is cookie expiration. Throwing AuthenticationError to trigger immediate re-login.`);
+          throw new AuthenticationError(`Received 500 for activity ${activityId} - expired cookie`, 500);
+        } else if (error.response.status >= 500 && error.response.status < 600) {
+          // Real server outage (502/503/504), preserve cache and don't re-login
+          logger.error(`Real server outage ${error.response.status} - preserving local cache, not re-login.`);
+        }
       }
       if (attempt === maxRetries - 1) {
         logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
-        throw error;
+        // Don't throw on network/timeout errors, just return null to preserve cache
+        return null;
       }
       await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
     }
@@ -270,7 +155,6 @@ async function getActivityDetailsRaw(
  * @param activityId - The ID of the activity to fetch.
  * @param userName - URL-encoded username.
  * @param userPwd - URL-encoded password.
- * @param templateFileName - Name of the login template file.
  * @param forceLogin - If true, bypasses cached cookie and forces a new login.
  * @returns The parsed JSON object of activity details, or null on failure.
  */
@@ -278,32 +162,21 @@ export async function fetchActivityData(
   activityId: string,
   userName: string,
   userPwd: string,
-  templateFileName: string = "login_template.txt",
+  forceLogin: boolean = false,
-  forceLogin: boolean = false
+  signal?: AbortSignal
-): Promise < any | null > {
+): Promise<any | null> {
-  let currentCookie = forceLogin ? null : await loadCachedCookie();
+  let currentCookie = forceLogin ? null : await getCachedCookieString();
 
   if (forceLogin && currentCookie) {
+    logger.info('Forcing new login. Clearing cached cookie.');
     await clearCookieCache();
     currentCookie = null;
   }
 
-  if (currentCookie) {
-    const isValid = await testCookieValidity(currentCookie);
-    if (!isValid) {
-      logger.info("Cached cookie test failed or cookie expired. Clearing cache.");
-      await clearCookieCache();
-      currentCookie = null;
-    } else {
-      logger.info("Using valid cached cookie.");
-    }
-  }
-
   if (!currentCookie) {
-    logger.info(forceLogin ? "Forcing new login." : "No valid cached cookie found or cache bypassed. Attempting login...");
+    logger.info('No cached cookie found. Attempting login...');
     try {
-      currentCookie = await getCompleteCookies(userName, userPwd, resolve(import.meta.dir, templateFileName));
+      currentCookie = await getCompleteCookies(userName, userPwd);
-      await saveCookieToCache(currentCookie);
     } catch (loginError) {
       logger.error(`Login process failed: ${(loginError as Error).message}`);
       return null;
@@ -311,12 +184,17 @@ export async function fetchActivityData(
   }
 
   if (!currentCookie) {
-    logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data.");
+    logger.error('Critical: No cookie available after login attempt. Cannot fetch activity data.');
     return null;
   }
 
+
+  logger.debug('Using cached cookie for API request.');
+
   try {
-    const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie);
+    logger.debug(`Calling getActivityDetailsRaw for activity ${activityId}...`);
+    const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie, 3, 10000, signal);
+    logger.debug(`getActivityDetailsRaw returned for activity ${activityId}`);
     if (rawActivityDetailsString) {
       const parsedOuter = JSON.parse(rawActivityDetailsString);
       return JSON.parse(parsedOuter.d);
@@ -324,17 +202,28 @@ export async function fetchActivityData(
     logger.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`);
     return null;
   } catch (error) {
+    if (signal?.aborted) {
+      logger.debug(`Activity ${activityId} fetch aborted.`);
+      return null;
+    }
     if (error instanceof AuthenticationError) {
-      logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`);
+      // Throttle: prevent thundering herd from multiple 500 errors
+      if (!tryAcquireAuthLock()) {
+        logger.info(`Auth throttled for activity ${activityId}. Reusing current cookies — likely still valid.`);
+        return null;
+      }
+
+      // Backup cookies before clearing so we can restore on re-login failure
+      backupCookies();
       await clearCookieCache();
 
       try {
-        logger.info("Attempting re-login due to authentication failure...");
+        logger.info('Attempting re-login due to authentication failure...');
-        currentCookie = await getCompleteCookies(userName, userPwd, resolve(import.meta.dir, templateFileName));
+        currentCookie = await getCompleteCookies(userName, userPwd);
-        await saveCookieToCache(currentCookie);
+        releaseAuthCooldown();
 
-        logger.info("Re-login successful. Retrying request for activity details once...");
+        logger.info('Re-login successful. Retrying request for activity details...');
-        const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie);
+        const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie, 1, 10000, signal);
         if (rawActivityDetailsStringRetry) {
           const parsedOuterRetry = JSON.parse(rawActivityDetailsStringRetry);
           return JSON.parse(parsedOuterRetry.d);
@@ -342,7 +231,9 @@ export async function fetchActivityData(
         logger.warn(`Still no details for activity ${activityId} after re-login and retry.`);
         return null;
       } catch (retryLoginOrFetchError) {
-        logger.error(`Error during re-login or retry fetch for activity ${activityId}: ${(retryLoginOrFetchError as Error).message}`);
+        logger.error(`Re-login or retry failed for activity ${activityId}: ${(retryLoginOrFetchError as Error).message}`);
+        // Restore old cookies instead of leaving cache empty
+        await restoreCookieBackup();
         return null;
       }
     } else {
@@ -351,6 +242,3 @@ export async function fetchActivityData(
     }
   }
 }
-
-// Optionally
-//export { clearCookieCache,testCookieValidity };

example.env

@@ -3,16 +3,42 @@ API_PASSWORD=
 PORT=3000
 FIXED_STAFF_ACTIVITY_ID=7095
 ALLOWED_ORIGINS=*
-S3_ENDPOINT= 
+S3_ENDPOINT=
+S3_PUBLIC_URL=
 S3_BUCKET_NAME=
 S3_ACCESS_KEY_ID=
 S3_SECRET_ACCESS_KEY=
 S3_REGION=
 S3_PUBLIC_URL_PREFIX=files
 REDIS_URL=redis://:dsas-cca@redis:6379
+LOG_LEVEL=info # Example: 'debug', 'info', 'warn', 'error'
+
+# ============================================================================
+# CRAWLER CONCURRENCY CONFIGURATION
+# ============================================================================
 MIN_ACTIVITY_ID_SCAN=3000
 MAX_ACTIVITY_ID_SCAN=8000
-CONCURRENT_API_CALLS=16
+
+# Maximum concurrent API calls during crawling (default: 8)
+# Higher values = faster crawling but more server load
+CONCURRENT_API_CALLS=8
+
+# Request timeout in milliseconds (default: 25000 = 25 seconds)
+CRAWLER_REQUEST_TIMEOUT_MS=25000
+
+# Maximum retries per request on transient errors (default: 3)
+CRAWLER_MAX_RETRIES=3
+
+# Delay between retries in milliseconds (default: 1000 = 1 second)
+CRAWLER_RETRY_DELAY_MS=1000
+
+# Rate limit: maximum requests per minute (default: unlimited)
+# Set to 0 for no limit
+CRAWLER_REQUESTS_PER_MINUTE=0
 STAFF_UPDATE_INTERVAL_MINS=360
 CLUB_UPDATE_INTERVAL_MINS=360
-LOG_LEVEL=info # Example: 'debug', 'info', 'warn', 'error'
+
+# Cache TTL Configuration (in seconds)
+ACTIVITY_CACHE_TTL=86400 # 24 hours for normal activity data
+STAFF_CACHE_TTL=86400 # 24 hours for staff data
+ERROR_CACHE_TTL=3600 # 1 hour for error states (allows retry)

index.ts

@@ -2,6 +2,24 @@
 import express, { Request, Response } from 'express';
 import { config } from 'dotenv';
 import cors from 'cors';
+import http from 'http';
+import https from 'https';
+import axios from 'axios';
+
+// Configure HTTP connection pooling with keep-alive
+axios.defaults.httpAgent = new http.Agent({
+  keepAlive: true,
+  maxSockets: 50,
+  maxFreeSockets: 10,
+  timeout: 30000
+});
+
+axios.defaults.httpsAgent = new https.Agent({
+  keepAlive: true,
+  maxSockets: 50,
+  maxFreeSockets: 10,
+  timeout: 30000
+});
 import { fetchActivityData } from './engage-api/get-activity';
 import { structActivityData } from './engage-api/struct-activity';
 import { structStaffData } from './engage-api/struct-staff';
@@ -20,8 +38,7 @@ import { extractBase64Image } from './utils/image-processor';
 import {
   initializeClubCache,
   updateStaleClubs,
-  initializeOrUpdateStaffCache,
+  initializeOrUpdateStaffCache
-  cleanupOrphanedS3Images
 } from './services/cache-manager';
 import { logger } from './utils/logger';
 import type { ActivityData } from './models/activity'
@@ -48,6 +65,10 @@ config();
 const USERNAME = process.env.API_USERNAME;
 const PASSWORD = process.env.API_PASSWORD;
 const PORT = process.env.PORT || 3000;
+
+// Mutex flags to prevent overlapping cron runs
+let isUpdatingClubs = false;
+let isUpdatingStaff = false;
 const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID;
 const allowedOriginsEnv = process.env.ALLOWED_ORIGINS || '*';
 const CLUB_CHECK_INTERVAL_SECONDS = parseInt(process.env.CLUB_CHECK_INTERVAL_SECONDS || '300', 10);
@@ -169,9 +190,11 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
       return res.json({});
     }
 
-    const allActivities = await Promise.all(
+    const allActivities: (ActivityData | null)[] = [];
-      activityKeys.map(async k => getActivityData(k.substring(ACTIVITY_KEY_PREFIX.length)))
+    for (const k of activityKeys) {
-    );
+      const activityData = await getActivityData(k.substring(ACTIVITY_KEY_PREFIX.length));
+      allActivities.push(activityData);
+    }
 
     /* ---------- gather available filter values for validation ---------- */
     const availableCategories    = new Set<string>();
@@ -239,13 +262,13 @@ app.get('/v1/activity/category', async (_req: Request, res: Response) => {
       logger.info('No activity keys found in Redis for categories.');
       return res.json({});
     }
-    // Fetch all activity data in parallel
+    // Fetch all activity data sequentially
-    const allActivityDataPromises = activityKeys.map(async (key) => {
+    const allActivities: (ActivityData | null)[] = [];
+    for (const key of activityKeys) {
       const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
-      return getActivityData(activityId);
+      const activityData = await getActivityData(activityId);
-    });
+      allActivities.push(activityData);
-
+    }
-    const allActivities = await Promise.all(allActivityDataPromises);
 
     allActivities.forEach((activityData: ActivityData | null) => {
       if (activityData && 
@@ -279,13 +302,13 @@ app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {
       logger.info('No activity keys found in Redis for academic years.');
       return res.json({});
     }
-    // 1. Fetch all activity data in parallel
+    // 1. Fetch all activity data sequentially
-    const allActivities = await Promise.all(
+    const allActivities: (ActivityData | null)[] = [];
-      activityKeys.map(async (key) => {
+    for (const key of activityKeys) {
-        const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
+      const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
-        return getActivityData(activityId);
+      const activityData = await getActivityData(activityId);
-      })
+      allActivities.push(activityData);
-    );
+    }
     // 2. Count activities per academic year
     allActivities.forEach((activityData: ActivityData | null) => {
       if (
@@ -397,13 +420,34 @@ async function performBackgroundTasks(): Promise<void> {
   try {
     await initializeClubCache();
     await initializeOrUpdateStaffCache(true);
-    await cleanupOrphanedS3Images();
+    // NOTE: Removed immediate cleanupOrphanedS3Images() call.
+    // Cleanup will run during periodic updateStaleClubs() instead.
+    // Running cleanup immediately after initialization caused race condition
+    // where newly uploaded images were deleted before they could be referenced.
     
     logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`);
-    setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000);
+    setInterval(() => {
+      if (isUpdatingClubs) {
+        logger.warn('Previous club update still running, skipping this run');
+        return;
+      }
+      isUpdatingClubs = true;
+      updateStaleClubs().finally(() => {
+        isUpdatingClubs = false;
+      });
+    }, CLUB_CHECK_INTERVAL_SECONDS * 1000);
 
     logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`);
-    setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000);
+    setInterval(() => {
+      if (isUpdatingStaff) {
+        logger.warn('Previous staff update still running, skipping this run');
+        return;
+      }
+      isUpdatingStaff = true;
+      initializeOrUpdateStaffCache(false).finally(() => {
+        isUpdatingStaff = false;
+      });
+    }, STAFF_CHECK_INTERVAL_SECONDS * 1000);
     
     logger.info('Background initialization and periodic task setup complete.');
   } catch (error) {

package.json

@@ -3,21 +3,27 @@
   "private": true,
   "scripts": {
     "start": "bun run index.ts",
-    "dev": "bun run --watch index.ts"
+    "dev": "bun run --watch index.ts",
+    "playwright:install": "bunx playwright install",
+    "cookie:get": "bun run test/get-cookies.ts",
+    "test": "bun test",
+    "test:auth": "bun test test/auth.spec.ts",
+    "test:ui": "bunx playwright test --ui"
   },
   "devDependencies": {
-    "@types/bun": "latest"
+    "@types/bun": "latest",
+    "typescript-language-server": "^5.1.3"
   },
   "peerDependencies": {
-    "typescript": "^5"
+    "typescript": "^5.9.3"
   },
   "dependencies": {
+    "@playwright/test": "^1.49.0",
     "axios": "^1.9.0",
     "cors": "^2.8.5",
     "crypto": "^1.0.1",
     "dotenv": "^16.5.0",
     "express": "^5.1.0",
-    "p-limit": "^6.2.0",
     "pangu": "^4.0.7",
     "sharp": "^0.34.1",
     "uuid": "^11.1.0"

playwright.config.ts

@@ -0,0 +1,24 @@
+import { defineConfig, type PlaywrightTestConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './test',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: 'list',
+  use: {
+    baseURL: 'https://engage.nkcswx.cn',
+    trace: 'on-first-retry',
+    headless: true,
+  },
+  timeout: 60000,
+  expect: {
+    timeout: 5000,
+  },
+  webServer: {
+    command: 'echo "No web server needed"',
+    port: 0,
+    reuseExistingServer: true,
+  },
+});

services/cache-manager.ts

@@ -1,6 +1,5 @@
 // services/cache-manager.ts
 import { config } from 'dotenv';
-import pLimit from 'p-limit';
 import { fetchActivityData } from '../engage-api/get-activity';
 import { structActivityData } from '../engage-api/struct-activity';
 import { structStaffData } from '../engage-api/struct-staff';
@@ -12,9 +11,10 @@ import {
   getAllActivityKeys,
   ACTIVITY_KEY_PREFIX
 } from './redis-service';
-import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service';
+import { uploadImageFromBase64, listS3Objects, constructS3Url } from './s3-service';
 import { extractBase64Image } from '../utils/image-processor';
 import { logger } from '../utils/logger';
+import { BatchProcessor } from '../utils/semaphore';
 
 import type { ActivityData } from '../models/activity';
 
@@ -25,36 +25,78 @@ const USERNAME = process.env.API_USERNAME;
 const PASSWORD = process.env.API_PASSWORD;
 const MIN_ACTIVITY_ID_SCAN = parseInt(process.env.MIN_ACTIVITY_ID_SCAN || '0', 10);
 const MAX_ACTIVITY_ID_SCAN = parseInt(process.env.MAX_ACTIVITY_ID_SCAN || '9999', 10);
-const CONCURRENT_API_CALLS = parseInt(process.env.CONCURRENT_API_CALLS || '10', 10);
 const CLUB_UPDATE_INTERVAL_MINS = parseInt(process.env.CLUB_UPDATE_INTERVAL_MINS || '60', 10);
 const STAFF_UPDATE_INTERVAL_MINS = parseInt(process.env.STAFF_UPDATE_INTERVAL_MINS || '60', 10);
 const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID;
 const S3_IMAGE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').replace(/\/$/, '');
 
-// Limit concurrent API calls
+// Crawler concurrency configuration
-const limit = pLimit(CONCURRENT_API_CALLS);
+const CONCURRENT_API_CALLS = parseInt(process.env.CONCURRENT_API_CALLS || '8', 10);
+const CRAWLER_REQUEST_TIMEOUT_MS = parseInt(process.env.CRAWLER_REQUEST_TIMEOUT_MS || '25000', 10);
+const CRAWLER_MAX_RETRIES = parseInt(process.env.CRAWLER_MAX_RETRIES || '3', 10);
+const CRAWLER_RETRY_DELAY_MS = parseInt(process.env.CRAWLER_RETRY_DELAY_MS || '1000', 10);
+
+// Module-level counter for skipped activities (reset at start of each scan)
+let skippedCount = 0;
 
 /**
  * Process and cache a single activity
  * @param activityId - The activity ID to process
+ * @param forceUpdate - If true, update cache even on fetch failure (default: false)
  * @returns The processed activity data
  */
-async function processAndCacheActivity(activityId: string): Promise<ActivityData> {
+async function processAndCacheActivity(activityId: string, forceUpdate: boolean = false): Promise<ActivityData> {
   logger.debug(`Processing activity ID: ${activityId}`);
   try {
     if (!USERNAME || !PASSWORD) {
       throw new Error('API username or password not configured');
     }
     
-    const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
+    // Add timeout protection via AbortController - properly cancels orphaned fetches
+    logger.debug(`Fetching activity data for ID: ${activityId}`);
+    const controller = new AbortController();
+    const timeoutId = setTimeout(
+      () => controller.abort(),
+      CRAWLER_REQUEST_TIMEOUT_MS + 5000
+    );
+
+    let activityJson: any = null;
+    try {
+      activityJson = await fetchActivityData(
+        activityId,
+        USERNAME,
+        PASSWORD,
+        false,
+        controller.signal
+      );
+    } finally {
+      clearTimeout(timeoutId);
+    }
+
+    if (controller.signal.aborted) {
+      logger.warn(`Request for activity ${activityId} timed out after ${CRAWLER_REQUEST_TIMEOUT_MS + 5000}ms. Cancelling orphaned fetch.`);
+      // Preserve existing cache on timeout
+      const existingData = await getActivityData(activityId);
+      return existingData || { lastCheck: new Date().toISOString(), error: `Timeout after ${CRAWLER_REQUEST_TIMEOUT_MS + 5000}ms` };
+    }
     let structuredActivity: ActivityData;
 
     if (!activityJson) {
-      logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`);
+      // CRITICAL: Only cache empty data if forceUpdate is true
-      structuredActivity = { 
+      // This prevents 5xx errors from overwriting valid local data
-        lastCheck: new Date().toISOString(), 
+      if (forceUpdate) {
-        source: 'api-fetch-empty' 
+        logger.info(`No data found for activity ID ${activityId} from engage API. Force updating cache.`);
-      };
+        structuredActivity = { 
+          lastCheck: new Date().toISOString(), 
+          source: 'api-fetch-empty' 
+        };
+        await setActivityData(activityId, structuredActivity);
+        return structuredActivity;
+      } else {
+        logger.warn(`No data for activity ${activityId}. Preserving existing cache - NOT updating.`);
+        const existingData = await getActivityData(activityId);
+        return existingData || { lastCheck: new Date().toISOString(), source: 'cache-preserved' };
+      }
     } else {
       structuredActivity = await structActivityData(activityJson);
       if (structuredActivity && structuredActivity.photo && 
@@ -82,70 +124,148 @@ async function processAndCacheActivity(activityId: string): Promise<ActivityData
     return structuredActivity;
   } catch (error) {
     logger.error(`Error processing activity ID ${activityId}:`, error);
-    const errorData: ActivityData = { 
+    // CRITICAL: On error, preserve existing cache instead of overwriting with error data
-      lastCheck: new Date().toISOString(), 
+    if (forceUpdate) {
-      error: "Failed to fetch or process" 
+      const errorData: ActivityData = { 
-    };
+        lastCheck: new Date().toISOString(), 
-    await setActivityData(activityId, errorData);
+        error: "Failed to fetch or process" 
-    return errorData;
+      };
+      await setActivityData(activityId, errorData);
+      return errorData;
+    } else {
+      logger.warn(`Error fetching activity ${activityId}. Preserving existing cache.`);
+      const existingData = await getActivityData(activityId);
+      return existingData || { lastCheck: new Date().toISOString(), error: (error as Error).message };
+    }
+  }
+}
+
+/**
+ * Process a single activity for initialization
+ * @param activityId - The activity ID to process
+ */
+async function processSingleActivity(activityId: string): Promise<void> {
+  const cachedData = await getActivityData(activityId);
+  
+  if (!cachedData || 
+    Object.keys(cachedData).length === 0 || 
+    !cachedData.lastCheck || 
+    cachedData.error) {
+    
+    logger.debug(`Initializing cache for activity ID: ${activityId}`);
+    await processAndCacheActivity(activityId);
+  } else {
+    skippedCount++;
   }
 }
 
 /**
  * Initialize the club cache by scanning through all activity IDs
+ * Processed concurrently with controlled parallelism
  */
 export async function initializeClubCache(): Promise<void> {
   logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`);
-  const promises: Promise<void>[] = [];
+  logger.info(`Concurrency: ${CONCURRENT_API_CALLS} parallel requests`);
   
-  for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) {
+  const totalIds = MAX_ACTIVITY_ID_SCAN - MIN_ACTIVITY_ID_SCAN + 1;
-    const activityId = String(i);
+  let successCount = 0;
-    promises.push(limit(async () => {
+  let errorCount = 0;
-      const cachedData = await getActivityData(activityId);
+  skippedCount = 0; // Reset for this run
-      if (!cachedData || 
+  
-        Object.keys(cachedData).length === 0 || 
+  // Generate array of activity IDs
-        !cachedData.lastCheck || 
+  const activityIds = Array.from(
-        cachedData.error) {
+    { length: totalIds },
-        logger.debug(`Initializing cache for activity ID: ${activityId}`);
+    (_, i) => String(MIN_ACTIVITY_ID_SCAN + i)
-        await processAndCacheActivity(activityId);
+  );
+  
+  // Create batch processor with concurrency control
+  const processor = new BatchProcessor(
+    async (activityId: string) => {
+      await processSingleActivity(activityId);
+      return activityId;
+    },
+    CONCURRENT_API_CALLS,
+    {
+      onError: (error, activityId) => {
+        errorCount++;
+        logger.error(`Error processing activity ID ${activityId}:`, error);
+      },
+      onProgress: (completed, total) => {
+        if (completed % 100 === 0 || completed === total) {
+          const mem = process.memoryUsage();
+          logger.info(`Progress: ${completed}/${total} (${Math.round(completed/total*100)}%) - Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount} | Heap: ${Math.round(mem.heapUsed/1024/1024)}MB | Concurrent: ${CONCURRENT_API_CALLS}`);
+        }
       }
-    }));
+    }
-  }
+  );
-
+  
-  await Promise.all(promises);
+  // Process all activities concurrently
-  logger.info('Initial club cache population finished.');
+  const results = await processor.process(activityIds);
+  successCount = results.length;
+  
+  logger.info(`Initial club cache population finished.`);
+  logger.info(`Summary: Total: ${totalIds}, Processed: ${activityIds.length}, Success: ${successCount}, Skipped: ${skippedCount}, Errors: ${errorCount}`);
 }
 
 /**
  * Update stale clubs in the cache
+ * Processed concurrently with controlled parallelism
  */
 export async function updateStaleClubs(): Promise<void> {
   logger.info('Starting stale club check...');
+  logger.info(`Concurrency: ${CONCURRENT_API_CALLS} parallel requests`);
   const now = Date.now();
   const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000;
-  const promises: Promise<void>[] = [];
   const activityKeys = await getAllActivityKeys();
-
+  
+  // Identify stale activities
+  const staleActivityIds: string[] = [];
   for (const key of activityKeys) {
     const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
-    promises.push(limit(async () => {
+    const cachedData = await getActivityData(activityId);
-      const cachedData = await getActivityData(activityId);
+    
-      
+    const needsUpdate = !cachedData || 
-      if (cachedData && cachedData.lastCheck) {
+                       Object.keys(cachedData).length === 0 ||
-        const lastCheckTime = new Date(cachedData.lastCheck).getTime();
+                       (!cachedData.lastCheck && !cachedData.error) ||
-        if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) {
+                       (cachedData.lastCheck && (now - new Date(cachedData.lastCheck).getTime()) > updateIntervalMs) ||
-          logger.info(`Activity ${activityId} is stale or had error. Updating...`);
+                       cachedData.error;
-          await processAndCacheActivity(activityId);
+    
-        }
+    if (needsUpdate) {
-      } else if (!cachedData || Object.keys(cachedData).length === 0) {
+      staleActivityIds.push(activityId);
-        logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`);
+    }
-        await processAndCacheActivity(activityId);
-      }
-    }));
   }
-
+  
-  await cleanupOrphanedS3Images();
+  if (staleActivityIds.length === 0) {
-  await Promise.all(promises);
+    logger.info('No stale activities found. Skipping update.');
+    logger.info('Stale club check finished.');
+    return;
+  }
+  
+  logger.info(`Found ${staleActivityIds.length} stale activities to update.`);
+  
+  // Create batch processor for concurrent updates
+  const processor = new BatchProcessor(
+    async (activityId: string) => {
+      logger.debug(`Updating stale activity ${activityId}`);
+      await processAndCacheActivity(activityId);
+      return activityId;
+    },
+    CONCURRENT_API_CALLS,
+    {
+      onError: (error, activityId) => {
+        logger.error(`Error updating stale activity ${activityId}:`, error);
+      },
+      onProgress: (completed, total) => {
+        if (completed % 10 === 0 || completed === total) {
+          logger.info(`Update progress: ${completed}/${total} (${Math.round(completed/total*100)}%)`);
+        }
+      }
+    }
+  );
+  
+  // Process stale activities concurrently
+  await processor.process(staleActivityIds);
+  
   logger.info('Stale club check finished.');
 }
 
@@ -194,59 +314,3 @@ export async function initializeOrUpdateStaffCache(forceUpdate: boolean = false)
     logger.error('Error initializing or updating staff cache:', error);
   }
 }
-
-/**
- * Clean up orphaned S3 images
- */
-export async function cleanupOrphanedS3Images(): Promise<void> {
-  logger.info('Starting S3 orphan image cleanup...');
-  const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : '';
-
-  try {
-    const referencedS3Urls = new Set<string>();
-    const allActivityRedisKeys = await getAllActivityKeys();
-    const S3_ENDPOINT = process.env.S3_ENDPOINT;
-
-    for (const redisKey of allActivityRedisKeys) {
-      const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length);
-      const activityData = await getActivityData(activityId);
-      
-      if (activityData && 
-        typeof activityData.photo === 'string' && 
-        activityData.photo.startsWith('http') && 
-        S3_ENDPOINT && 
-        activityData.photo.startsWith(S3_ENDPOINT)) {
-        referencedS3Urls.add(activityData.photo);
-      }
-    }
-    
-    logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`);
-
-    const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix);
-    if (!s3ObjectKeys || s3ObjectKeys.length === 0) {
-      logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`);
-      return;
-    }
-    
-    logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`);
-
-    const orphanedObjectKeys: string[] = [];
-    for (const objectKey of s3ObjectKeys) {
-      const s3Url = constructS3Url(objectKey);
-      if (s3Url && !referencedS3Urls.has(s3Url)) {
-        orphanedObjectKeys.push(objectKey);
-      }
-    }
-
-    if (orphanedObjectKeys.length > 0) {
-      logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`);
-      await deleteS3Objects(orphanedObjectKeys);
-    } else {
-      logger.info('No orphaned S3 images found after comparison.');
-    }
-
-    logger.info('S3 orphan image cleanup finished.');
-  } catch (error) {
-    logger.error('Error during S3 orphan image cleanup:', error);
-  }
-}

services/playwright-auth.ts

@@ -0,0 +1,280 @@
+import { chromium, type BrowserContext, type Cookie } from 'playwright';
+import { logger } from '../utils/logger';
+import * as fs from 'node:fs';
+import { resolve } from 'node:path';
+
+const LOGIN_URL = 'https://engage.nkcswx.cn/Login.aspx';
+const COOKIE_FILE_PATH = resolve(import.meta.dir, 'cookies.json');
+
+let _inMemoryCookies: Cookie[] | null = null;
+
+// Login lock to prevent concurrent login attempts
+let _loginLock: Promise<Cookie[]> | null = null;
+
+// Cookie backup: preserved before clearCookieCache, restored on re-login failure
+let _cookieBackup: Cookie[] | null = null;
+
+// Auth failure throttle: debounce consecutive re-login triggers from 500 errors
+// Prevents thundering herd when server is slow and returns many 500s
+let _authFailureCooldownUntil = 0;
+const AUTH_FAILURE_COOLDOWN_MS = 15000; // 15s cooldown between re-login cycles
+
+/**
+ * Put all callers to wait during auth cooldown window.
+ * Returns true if auth is allowed (outside cooldown), false if throttled.
+ */
+export function tryAcquireAuthLock(): boolean {
+  const now = Date.now();
+  if (now < _authFailureCooldownUntil) {
+    const remaining = _authFailureCooldownUntil - now;
+    logger.warn(
+      `Re-login throttled: ${Math.round(remaining / 1000)}s cooldown remaining. ` +
+      `Existing cookies are likely still valid — server 500 is a temporary slowdown.`
+    );
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Called after a successful re-login to release the cooldown.
+ */
+export function releaseAuthCooldown(): void {
+  _authFailureCooldownUntil = Date.now() + AUTH_FAILURE_COOLDOWN_MS;
+  logger.info(`Auth cooldown set: ${AUTH_FAILURE_COOLDOWN_MS}ms to prevent thundering herd re-logins.`);
+}
+
+/**
+ * Ensure only one login process runs at a time
+ */
+export async function ensureSingleLogin(username: string, password: string): Promise<Cookie[]> {
+  // Wait for any existing login to complete
+  if (_loginLock) {
+    logger.info('Login in progress, waiting for existing login to complete...');
+    await _loginLock;
+  }
+  
+  // Create new lock promise for this login attempt
+  _loginLock = (async () => {
+    try {
+      return await loginWithPlaywright(username, password);
+    } finally {
+      _loginLock = null;
+    }
+  })();
+  
+  return await _loginLock;
+}
+
+/**
+ * Login using Playwright and extract cookies
+ */
+export async function loginWithPlaywright(username: string, password: string): Promise<Cookie[]> {
+  logger.info('Starting Playwright login process...');
+  
+  const browserLaunchOptions: any = { 
+    headless: true,
+    args: ['--no-sandbox', '--disable-setuid-sandbox']
+  };
+
+  const browser = await chromium.launch(browserLaunchOptions);
+
+  try {
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    });
+
+    const page = await context.newPage();
+    
+    logger.info(`Navigating to login page: ${LOGIN_URL}`);
+    await page.goto(LOGIN_URL, { 
+      waitUntil: 'networkidle',
+      timeout: 30000
+    });
+
+    logger.info('Login page loaded. Filling form...');
+
+    const usernameField = page.locator('input[name="ctl00$PageContent$loginControl$txtUN"]');
+    await usernameField.fill(decodeURIComponent(username));
+
+    const passwordField = page.locator('input[name="ctl00$PageContent$loginControl$txtPwd"]');
+    await passwordField.fill(decodeURIComponent(password));
+
+    const rememberMe = page.locator('input[name="ctl00$PageContent$loginControl$cbRememberMe"]');
+    await rememberMe.check().catch(() => {
+      logger.debug('Could not check remember me checkbox (optional)');
+    });
+
+    const loginButton = page.locator('input[name="ctl00$PageContent$loginControl$btnLogin"]');
+    logger.info('Clicking login button...');
+    await loginButton.click();
+
+    await page.waitForLoadState('networkidle', { timeout: 30000 });
+    
+    const isLoggedIn = await checkLoginSuccess(page);
+    
+    if (!isLoggedIn) {
+      const errorMessage = await page.locator('.error, .errorMessage, [class*="error"]').first().textContent();
+      throw new Error(`Login failed. Possible error: ${errorMessage || 'Unknown error'}`);
+    }
+
+    logger.info('Login successful! Extracting cookies...');
+
+    const cookies = await context.cookies();
+    logger.info(`Extracted ${cookies.length} cookies`);
+
+    await saveCookiesToCache(cookies);
+    logImportantCookies(cookies);
+
+    await browser.close();
+    return cookies;
+  } catch (error) {
+    logger.error('Error during Playwright login:', error);
+    await browser.close();
+    throw error;
+  }
+}
+
+/**
+ * Check if login was successful
+ */
+async function checkLoginSuccess(page: any): Promise<boolean> {
+  await page.waitForTimeout(1000);
+
+  const currentUrl = page.url();
+  const notOnLoginPage = !currentUrl.includes('Login.aspx');
+
+  const hasLogoutLink = await page.locator('text=Logout, text=退出，text=Sign Out').count() > 0;
+  const hasWelcomeText = await page.locator('text=Welcome, text=欢迎').count() > 0;
+
+  return notOnLoginPage || hasLogoutLink || hasWelcomeText;
+}
+
+/**
+ * Log important cookies for debugging
+ */
+function logImportantCookies(cookies: Cookie[]): void {
+  const importantCookieNames = [
+    'ASP.NET_SessionId',
+    '.ASPXFORMSAUTH',
+  ];
+
+  logger.debug('Important cookies:');
+  cookies.forEach(cookie => {
+    if (importantCookieNames.some(name => cookie.name.includes(name))) {
+      logger.debug(`  ${cookie.name}: ${cookie.value.substring(0, 50)}${cookie.value.length > 50 ? '...' : ''}`);
+    }
+  });
+}
+
+/**
+ * Load cookies from cache file
+ */
+export async function loadCachedCookies(): Promise<Cookie[] | null> {
+  if (_inMemoryCookies) {
+    logger.debug('Using in-memory cached cookies.');
+    return _inMemoryCookies;
+  }
+
+  if (!fs.existsSync(COOKIE_FILE_PATH)) {
+    logger.debug('Cookie cache file not found. No cached cookies loaded.');
+    return null;
+  }
+
+  try {
+    const cookies = JSON.parse(fs.readFileSync(COOKIE_FILE_PATH, 'utf-8')) as Cookie[];
+    _inMemoryCookies = cookies;
+    logger.debug(`Loaded ${cookies.length} cookies from file cache.`);
+    return cookies;
+  } catch (error: any) {
+    logger.warn('Error loading cookies from file:', error.message);
+    return null;
+  }
+}
+
+/**
+ * Save cookies to cache file
+ */
+export async function saveCookiesToCache(cookies: Cookie[]): Promise<void> {
+  if (!cookies || cookies.length === 0) {
+    logger.warn('Attempted to save empty or null cookies. Aborting save.');
+    return;
+  }
+
+  _inMemoryCookies = cookies;
+  
+  try {
+    await fs.promises.writeFile(COOKIE_FILE_PATH, JSON.stringify(cookies, null, 2), 'utf-8');
+    logger.debug('Cookies saved to file cache.');
+  } catch (error: any) {
+    logger.error('Error saving cookies to file:', error.message);
+  }
+}
+
+/**
+ * Backup current cookies before clearing. Restored if re-login fails.
+ */
+export function backupCookies(): Cookie[] | null {
+  if (_inMemoryCookies) {
+    _cookieBackup = [..._inMemoryCookies];
+    logger.info('Cookies backed up before clear.');
+  }
+  return _cookieBackup;
+}
+
+/**
+ * Restore cookies from backup after failed re-login.
+ */
+export async function restoreCookieBackup(): Promise<boolean> {
+  if (_cookieBackup) {
+    _inMemoryCookies = _cookieBackup;
+    try {
+      await fs.promises.writeFile(COOKIE_FILE_PATH, JSON.stringify(_cookieBackup, null, 2), 'utf-8');
+      logger.info('Cookies restored from backup successfully.');
+      _cookieBackup = null;
+      return true;
+    } catch (error: any) {
+      logger.error('Failed to restore cookies from backup:', error.message);
+      return false;
+    }
+  }
+  logger.warn('No cookie backup available for restore.');
+  return false;
+}
+
+/**
+ * Clear cookie cache
+ * Prefer backupAndClearCookieCache() instead to preserve old cookies.
+ */
+export async function clearCookieCache(): Promise<void> {
+  _inMemoryCookies = null;
+  
+  try {
+    await fs.promises.unlink(COOKIE_FILE_PATH);
+    logger.debug('Cookie cache file deleted.');
+  } catch (error: any) {
+    if (error.code !== 'ENOENT') {
+      logger.error('Error deleting cookie file:', error.message);
+    } else {
+      logger.debug('Cookie cache file did not exist, no need to delete.');
+    }
+  }
+}
+
+/**
+ * Convert cookies array to cookie string for axios
+ */
+export function cookiesToString(cookies: Cookie[]): string {
+  return cookies.map(c => `${c.name}=${c.value}`).join('; ');
+}
+
+/**
+ * Get cookie string from cache
+ */
+export async function getCachedCookieString(): Promise<string | null> {
+  const cookies = await loadCachedCookies();
+  if (!cookies || cookies.length === 0) {
+    return null;
+  }
+  return cookiesToString(cookies);
+}

services/redis-service.ts

@@ -8,6 +8,11 @@ config();
 export const ACTIVITY_KEY_PREFIX = 'activity:'; // Exported for use in cache-manager
 const STAFF_KEY = 'staffs:all';
 
+// Cache TTL configuration (in seconds)
+const ACTIVITY_CACHE_TTL = parseInt(process.env.ACTIVITY_CACHE_TTL || '86400', 10); // Default: 24 hours
+const STAFF_CACHE_TTL = parseInt(process.env.STAFF_CACHE_TTL || '86400', 10); // Default: 24 hours
+const ERROR_CACHE_TTL = parseInt(process.env.ERROR_CACHE_TTL || '3600', 10); // Default: 1 hour for errors
+
 // Always create a new client instance with .env config
 const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
 let redisClient: RedisClient | null = null;
@@ -39,17 +44,25 @@ export async function getActivityData(activityId: string): Promise<any | null> {
 }
 
 /**
- * Sets activity data in Redis.
+ * Sets activity data in Redis with TTL.
  * @param activityId - The activity ID to set
  * @param data - The activity data object
+ * @param ttl - Optional TTL in seconds (defaults to ACTIVITY_CACHE_TTL, or ERROR_CACHE_TTL if data has error)
  */
-export async function setActivityData(activityId: string, data: any): Promise<void> {
+export async function setActivityData(activityId: string, data: any, ttl?: number): Promise<void> {
   if (!redisClient) {
     logger.warn('Redis client not available, skipping setActivityData');
     return;
   }
   try {
-    await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data));
+    // Use shorter TTL for error states to allow retry
+    const expiration = data?.error ? ERROR_CACHE_TTL : (ttl || ACTIVITY_CACHE_TTL);
+    // Bun's RedisClient doesn't have setEx, use raw SETEX command
+    await redisClient.send('SETEX', [
+      `${ACTIVITY_KEY_PREFIX}${activityId}`,
+      String(expiration),
+      JSON.stringify(data)
+    ]);
   } catch (err) {
     logger.error(`Error setting activity ${activityId} in Redis:`, err);
   }
@@ -74,16 +87,23 @@ export async function getStaffData(): Promise<any | null> {
 }
 
 /**
- * Sets staff data in Redis.
+ * Sets staff data in Redis with TTL.
  * @param data - The staff data object
+ * @param ttl - Optional TTL in seconds (defaults to STAFF_CACHE_TTL)
  */
-export async function setStaffData(data: any): Promise<void> {
+export async function setStaffData(data: any, ttl?: number): Promise<void> {
   if (!redisClient) {
     logger.warn('Redis client not available, skipping setStaffData');
     return;
   }
   try {
-    await redisClient.set(STAFF_KEY, JSON.stringify(data));
+    const expiration = ttl || STAFF_CACHE_TTL;
+    // Use raw SETEX command for TTL support
+    await redisClient.send('SETEX', [
+      STAFF_KEY,
+      String(expiration),
+      JSON.stringify(data)
+    ]);
   } catch (err) {
     logger.error('Error setting staff data in Redis:', err);
   }
@@ -103,8 +123,11 @@ export async function getAllActivityKeys(): Promise<string[]> {
     // Using raw SCAN command since Bun's RedisClient doesn't have a scan method
     const keys: string[] = [];
     let cursor = '0';
+    let iteration = 0;
+    const MAX_ITERATIONS = 1000; // Safety limit to prevent infinite loops
     
     do {
+      iteration++;
       // Use send method to execute raw Redis commands
       const result = await redisClient.send('SCAN', [
         cursor, 
@@ -114,15 +137,24 @@ export async function getAllActivityKeys(): Promise<string[]> {
         '100'
       ]);
       
-      cursor = result[0];
+      // Force convert to string to ensure type consistency (Bun may return Buffer)
+      cursor = String(result[0] ?? '0');
       const foundKeys = result[1] || [];
       
+      logger.debug(`SCAN iteration ${iteration}: cursor=${cursor}, found ${foundKeys.length} keys, total=${keys.length + foundKeys.length}`);
+      
       // Add the found keys to our array
       keys.push(...foundKeys);
       
+      // Prevent infinite loop
+      if (iteration >= MAX_ITERATIONS) {
+        logger.warn(`SCAN reached max iterations (${MAX_ITERATIONS}). May have incomplete results.`);
+        break;
+      }
+      
     } while (cursor !== '0');
     
-    logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`);
+    logger.info(`Found ${keys.length} activity keys in Redis after ${iteration} SCAN iterations.`);
     return keys;
   } catch (err) {
     logger.error('Error getting all activity keys from Redis using SCAN:', err);

services/s3-service.ts

@@ -15,6 +15,7 @@ const S3_REGION = process.env.S3_REGION;
 const S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID;
 const S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY;
 const BUCKET_NAME = process.env.S3_BUCKET_NAME;
+const S3_PUBLIC_URL = process.env.S3_PUBLIC_URL;
 const PUBLIC_URL_FILE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').replace(/\/$/, '');
 
 // Initialize S3 client
@@ -148,53 +149,9 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
   }
 }
 
-/**
- * Deletes multiple objects from S3.
- * @param objectKeysArray - Array of object keys to delete
- * @returns True if successful or partially successful, false on major error
- */
-export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolean> {
-  if (!s3Client) {
-    logger.warn('S3 client not configured. Cannot delete objects.');
-    return false;
-  }
-  if (!objectKeysArray || objectKeysArray.length === 0) {
-    logger.info('No objects to delete from S3.');
-    return true;
-  }
-  try {
-    // With Bun's S3Client, we need to delete objects one by one
-    // Process in batches of 100 for better performance
-    const BATCH_SIZE = 100;
-    let successCount = 0;
-    let errorCount = 0;
-    
-    for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) {
-      const batch = objectKeysArray.slice(i, i + BATCH_SIZE);
-      // Process batch in parallel
-      const results = await Promise.allSettled(
-        batch.map(key => s3Client!.delete(key))
-      );
-      // Count successes and failures
-      for (const result of results) {
-        if (result.status === 'fulfilled') {
-          successCount++;
-        } else {
-          errorCount++;
-          logger.error(`Failed to delete object: ${result.reason}`);
-        }
-      }
-    }
-    logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
-    return errorCount === 0; // True if all succeeded
-  } catch (error) {
-    logger.error('S3 DeleteObjects Error:', error);
-    return false;
-  }
-}
-
 /**
  * Constructs the public S3 URL for an object key.
+ * Uses S3_PUBLIC_URL if set (reverse proxy scenario), otherwise uses S3_ENDPOINT.
  * @param objectKey - The key of the object in S3
  * @returns The full public URL
  */
@@ -202,8 +159,8 @@ export function constructS3Url(objectKey: string): string {
   if (!S3_ENDPOINT || !BUCKET_NAME) {
     return '';
   }
-  // Ensure S3_ENDPOINT does not end with a slash
+  // Use S3_PUBLIC_URL if set (reverse proxy), otherwise use S3_ENDPOINT
-  const s3Base = S3_ENDPOINT.replace(/\/$/, '');
+  const s3Base = (S3_PUBLIC_URL || S3_ENDPOINT).replace(/\/$/, '');
   // Ensure BUCKET_NAME does not start or end with a slash
   const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, '');
   // Ensure objectKey does not start with a slash

startup.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+set -e
+
+echo "🚀 Starting DSAS CCA Backend..."
+
+# Check if cookies exist and are valid
+if [ -f /usr/src/app/services/cookies.json ]; then
+  echo "📁 Cookies file found. Checking validity..."
+  
+  # Try to fetch a simple activity to test cookies
+  # If it fails, we'll get fresh cookies
+  if ! timeout 10 bun run test/test-cookies-validity.ts 2>/dev/null; then
+    echo "⚠️ Cookies are invalid or expired. Getting fresh cookies..."
+    bun run test/get-cookies.ts
+  else
+    echo "✅ Cookies are valid. Using cached cookies."
+  fi
+else
+  echo "📁 No cookies file found. Getting fresh cookies..."
+  bun run test/get-cookies.ts
+fi
+
+# Start the application
+echo "🎯 Starting application..."
+exec bun run index.ts

test/auth.spec.ts

@@ -0,0 +1,118 @@
+import { test, expect } from 'bun:test';
+import { chromium, type Cookie } from 'playwright';
+import * as fs from 'fs';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const COOKIE_FILE_PATH = resolve(__dirname, '../services/cookies.json');
+
+const testUsername = process.env.API_USERNAME || 'test@test.com';
+const testPassword = process.env.API_PASSWORD || 'test123';
+
+test('should login and extract cookies successfully', async () => {
+  const browser = await chromium.launch({ 
+    headless: true,
+    args: ['--no-sandbox', '--disable-setuid-sandbox']
+  });
+
+  try {
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    });
+
+    const page = await context.newPage();
+    
+    await page.goto('https://engage.nkcswx.cn/Login.aspx', { 
+      waitUntil: 'networkidle',
+      timeout: 60000
+    });
+
+    const usernameField = page.locator('input[name="ctl00$PageContent$loginControl$txtUN"]');
+    await usernameField.fill(decodeURIComponent(testUsername));
+
+    const passwordField = page.locator('input[name="ctl00$PageContent$loginControl$txtPwd"]');
+    await passwordField.fill(decodeURIComponent(testPassword));
+
+    const loginButton = page.locator('input[name="ctl00$PageContent$loginControl$btnLogin"]');
+    await loginButton.click();
+
+    await page.waitForLoadState('networkidle', { timeout: 60000 });
+
+    const cookies = await context.cookies();
+    
+    expect(cookies).toBeDefined();
+    expect(cookies.length).toBeGreaterThan(0);
+    
+    const hasSessionCookie = cookies.some(c => c.name === 'ASP.NET_SessionId');
+    expect(hasSessionCookie).toBe(true);
+
+    fs.writeFileSync(COOKIE_FILE_PATH, JSON.stringify(cookies, null, 2));
+  } finally {
+    await browser.close();
+  }
+}, 120000);
+
+test('should load cookies from file if exists', () => {
+  if (!fs.existsSync(COOKIE_FILE_PATH)) {
+    throw new Error('Cookie file does not exist');
+  }
+
+  const cookies = JSON.parse(fs.readFileSync(COOKIE_FILE_PATH, 'utf-8')) as Cookie[];
+  expect(cookies.length).toBeGreaterThan(0);
+});
+
+test('should test cookie validity', async () => {
+  if (!fs.existsSync(COOKIE_FILE_PATH)) {
+    throw new Error('Cookie file does not exist');
+  }
+
+  const cookies = JSON.parse(fs.readFileSync(COOKIE_FILE_PATH, 'utf-8')) as Cookie[];
+  
+  const browser = await chromium.launch({ 
+    headless: true,
+    args: ['--no-sandbox', '--disable-setuid-sandbox']
+  });
+
+  try {
+    const context = await browser.newContext();
+    await context.addCookies(cookies);
+    
+    const page = await context.newPage();
+    
+    await page.goto('https://engage.nkcswx.cn/', { 
+      waitUntil: 'networkidle',
+      timeout: 30000
+    });
+
+    const url = page.url();
+    const isRedirectedToLogin = url.includes('/Login.aspx');
+    
+    expect(isRedirectedToLogin).toBe(false);
+  } finally {
+    await browser.close();
+  }
+}, 60000);
+
+test('should convert cookies to string format', () => {
+  if (!fs.existsSync(COOKIE_FILE_PATH)) {
+    throw new Error('Cookie file does not exist');
+  }
+
+  const cookies = JSON.parse(fs.readFileSync(COOKIE_FILE_PATH, 'utf-8')) as Cookie[];
+  
+  const cookieString = cookies.map(c => `${c.name}=${c.value}`).join('; ');
+  expect(cookieString).toBeDefined();
+  expect(cookieString.length).toBeGreaterThan(0);
+  expect(cookieString).toContain('ASP.NET_SessionId=');
+});
+
+test('should clear cookie cache', () => {
+  if (fs.existsSync(COOKIE_FILE_PATH)) {
+    fs.unlinkSync(COOKIE_FILE_PATH);
+  }
+  
+  const exists = fs.existsSync(COOKIE_FILE_PATH);
+  expect(exists).toBe(false);
+});

test/get-cookies.ts

@@ -0,0 +1,24 @@
+import { loginWithPlaywright, saveCookiesToCache } from '../services/playwright-auth';
+
+const username = process.env.API_USERNAME;
+const password = process.env.API_PASSWORD;
+
+if (!username || !password) {
+  console.error('❌ API_USERNAME and API_PASSWORD environment variables are required');
+  process.exit(1);
+}
+
+console.log('🔑 Starting cookie extraction...\n');
+
+loginWithPlaywright(username, password)
+  .then(cookies => {
+    console.log(`\n✅ Extracted ${cookies.length} cookies`);
+    console.log('📁 Cookies saved to: ./services/cookies.json');
+    
+    saveCookiesToCache(cookies);
+    process.exit(0);
+  })
+  .catch(error => {
+    console.error('\n❌ Cookie extraction failed:', error);
+    process.exit(1);
+  });

test/test-concurrency.ts

@@ -0,0 +1,141 @@
+// test/test-concurrency.ts
+/**
+ * Test script for concurrency features
+ * Run with: bun run test/test-concurrency.ts
+ */
+
+import { Semaphore, executeWithConcurrency, BatchProcessor } from '../utils/semaphore';
+
+// Simulate API call
+function simulateApiCall(id: number, delay: number = 100): Promise<{ id: number; result: string }> {
+  return new Promise((resolve) => {
+    setTimeout(() => {
+      resolve({ id, result: `Result for ${id}` });
+    }, delay);
+  });
+}
+
+async function testSemaphore(): Promise<void> {
+  console.log('\n=== Test 1: Basic Semaphore ===');
+  const semaphore = new Semaphore(3);
+  
+  const start = Date.now();
+  const promises = [];
+  
+  for (let i = 1; i <= 10; i++) {
+    const id = i;
+    promises.push(
+      (async () => {
+        await semaphore.acquire();
+        console.log(`[${id}] Acquired permit (available: ${semaphore.getAvailablePermits()})`);
+        await simulateApiCall(id, 200);
+        console.log(`[${id}] Releasing permit`);
+        semaphore.release();
+      })()
+    );
+  }
+  
+  await Promise.all(promises);
+  const duration = Date.now() - start;
+  console.log(`\n✓ Completed 10 tasks with max 3 concurrent in ${duration}ms`);
+  console.log(`  (Sequential would take ~2000ms, parallel should be ~700-800ms)`);
+}
+
+async function testExecuteWithConcurrency(): Promise<void> {
+  console.log('\n=== Test 2: executeWithConcurrency ===');
+  
+  const tasks = Array.from({ length: 10 }, (_, i) => () => simulateApiCall(i + 1, 100));
+  
+  const start = Date.now();
+  const results = await executeWithConcurrency(tasks, 5);
+  const duration = Date.now() - start;
+  
+  console.log(`✓ Completed ${results.length} tasks with max 5 concurrent in ${duration}ms`);
+  console.log(`  Results: ${results.map(r => r.id).join(', ')}`);
+}
+
+async function testBatchProcessor(): Promise<void> {
+  console.log('\n=== Test 3: BatchProcessor ===');
+  
+  const items = Array.from({ length: 20 }, (_, i) => ({ id: i + 1, name: `Item ${i + 1}` }));
+  let processedCount = 0;
+  
+  const processor = new BatchProcessor(
+    async (item: { id: number; name: string }) => {
+      await simulateApiCall(item.id, 50);
+      processedCount++;
+      return { ...item, processed: true };
+    },
+    4, // concurrency
+    {
+      onProgress: (completed, total) => {
+        if (completed % 5 === 0 || completed === total) {
+          console.log(`  Progress: ${completed}/${total} (${Math.round(completed / total * 100)}%)`);
+        }
+      },
+      onError: (error, item) => {
+        console.error(`  Error processing item ${item.id}:`, error.message);
+      }
+    }
+  );
+  
+  const start = Date.now();
+  const results = await processor.process(items);
+  const duration = Date.now() - start;
+  
+  console.log(`✓ Processed ${results.length} items with max 4 concurrent in ${duration}ms`);
+  console.log(`  Expected ~500ms (20 items / 4 concurrent * 50ms each)`);
+}
+
+async function testErrorHandling(): Promise<void> {
+  console.log('\n=== Test 4: Error Handling ===');
+  
+  const items = [1, 2, 3, 4, 5];
+  let errorCount = 0;
+  
+  const processor = new BatchProcessor(
+    async (id: number) => {
+      if (id % 2 === 0) {
+        throw new Error(`Simulated error for ${id}`);
+      }
+      return { id, success: true };
+    },
+    3,
+    {
+      onError: (error, item) => {
+        errorCount++;
+        console.log(`  Caught error for item ${item}: ${error.message}`);
+      }
+    }
+  );
+  
+  const results = await processor.process(items);
+  console.log(`✓ Completed: ${results.length} success, ${errorCount} errors (errors handled gracefully)`);
+}
+
+async function main(): Promise<void> {
+  console.log('╔═══════════════════════════════════════════════════╗');
+  console.log('║         Concurrency Module Test Suite            ║');
+  console.log('╚═══════════════════════════════════════════════════╝');
+  
+  try {
+    await testSemaphore();
+    await testExecuteWithConcurrency();
+    await testBatchProcessor();
+    await testErrorHandling();
+    
+    console.log('\n╔═══════════════════════════════════════════════════╗');
+    console.log('║              All tests passed! ✓                  ║');
+    console.log('╚═══════════════════════════════════════════════════╝');
+    console.log('\n📝 Configuration:');
+    console.log('   - Set CONCURRENT_API_CALLS in .env to control parallelism');
+    console.log('   - Current default: 8 concurrent requests');
+    console.log('   - Example: CONCURRENT_API_CALLS=16 for faster crawling');
+    console.log('');
+  } catch (error) {
+    console.error('\n❌ Test failed:', error);
+    process.exit(1);
+  }
+}
+
+main();

test/test-cookies-validity.ts

@@ -0,0 +1,35 @@
+import axios from 'axios';
+
+const COOKIE_FILE = './services/cookies.json';
+
+async function testCookies() {
+  try {
+    const fs = await import('fs');
+    if (!fs.existsSync(COOKIE_FILE)) {
+      return false;
+    }
+
+    const cookies = JSON.parse(fs.readFileSync(COOKIE_FILE, 'utf-8'));
+    const cookieString = cookies.map((c: any) => `${c.name}=${c.value}`).join('; ');
+
+    const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
+    const headers = {
+      'Content-Type': 'application/json; charset=UTF-8',
+      'Cookie': cookieString,
+      'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA)',
+    };
+    const payload = { "activityID": "3350" };
+
+    await axios.post(url, payload, {
+      headers,
+      timeout: 10000
+    });
+
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+const isValid = await testCookies();
+process.exit(isValid ? 0 : 1);

utils/semaphore.ts

@@ -0,0 +1,220 @@
+// utils/semaphore.ts
+/**
+ * Semaphore implementation for controlling concurrent operations
+ * Based on patterns from civitai/civitai and p-queue
+ */
+
+export class Semaphore {
+  private capacity: number;
+  private permits: number;
+  private queue: Array<() => void> = [];
+
+  constructor(capacity: number) {
+    if (capacity < 1) {
+      throw new Error('Semaphore capacity must be at least 1');
+    }
+    this.capacity = capacity;
+    this.permits = capacity;
+  }
+
+  /**
+   * Acquire a permit. If none available, waits until one is released.
+   */
+  async acquire(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      if (this.permits > 0) {
+        this.permits--;
+        resolve();
+      } else {
+        // Queue the release callback
+        this.queue.push(() => {
+          resolve();
+        });
+      }
+    });
+  }
+
+  /**
+   * Release a permit and wake up a waiting task if any.
+   */
+  release(): void {
+    if (this.queue.length > 0) {
+      const next = this.queue.shift();
+      if (next) {
+        next();
+      }
+    } else {
+      this.permits++;
+    }
+  }
+
+  /**
+   * Get current available permits.
+   */
+  getAvailablePermits(): number {
+    return this.permits;
+  }
+
+  /**
+   * Get total capacity.
+   */
+  getCapacity(): number {
+    return this.capacity;
+  }
+
+  /**
+   * Get number of waiting tasks.
+   */
+  getWaitingCount(): number {
+    return this.queue.length;
+  }
+}
+
+/**
+ * Execute async tasks with concurrency limit
+ * @param tasks Array of async task functions
+ * @param concurrency Maximum concurrent tasks
+ * @returns Promise that resolves with all results when complete
+ */
+export async function executeWithConcurrency<T>(
+  tasks: Array<() => Promise<T>>,
+  concurrency: number
+): Promise<T[]> {
+  const semaphore = new Semaphore(concurrency);
+  const results: T[] = new Array(tasks.length);
+
+  const promises = tasks.map(async (task, index) => {
+    await semaphore.acquire();
+    try {
+      results[index] = await task();
+      return results[index];
+    } finally {
+      semaphore.release();
+    }
+  });
+
+  return Promise.all(promises);
+}
+
+/**
+ * Execute async tasks with concurrency limit and progress callback
+ * @param tasks Array of async task functions
+ * @param concurrency Maximum concurrent tasks
+ * @param onProgress Callback with (completed, total, result) after each task
+ * @returns Promise that resolves with all results when complete
+ */
+export async function executeWithConcurrencyAndProgress<T>(
+  tasks: Array<() => Promise<T>>,
+  concurrency: number,
+  onProgress?: (completed: number, total: number, result: T, error?: Error) => void
+): Promise<T[]> {
+  const semaphore = new Semaphore(concurrency);
+  const results: T[] = new Array(tasks.length);
+  let completed = 0;
+  const total = tasks.length;
+
+  const promises = tasks.map(async (task, index) => {
+    await semaphore.acquire();
+    try {
+      results[index] = await task();
+      completed++;
+      onProgress?.(completed, total, results[index]!);
+      return results[index];
+    } catch (error) {
+      completed++;
+      onProgress?.(completed, total, undefined as T, error as Error);
+      throw error;
+    } finally {
+      semaphore.release();
+    }
+  });
+
+  return Promise.all(promises);
+}
+
+/**
+ * Batch processor with concurrency control
+ * Useful for processing large arrays in chunks with controlled concurrency
+ */
+export class BatchProcessor<T, R> {
+  private semaphore: Semaphore;
+  private processor: (item: T, index: number) => Promise<R>;
+  private onError?: (error: Error, item: T, index: number) => void;
+  private onProgress?: (completed: number, total: number) => void;
+
+  constructor(
+    processor: (item: T, index: number) => Promise<R>,
+    concurrency: number,
+    options?: {
+      onError?: (error: Error, item: T, index: number) => void;
+      onProgress?: (completed: number, total: number) => void;
+    }
+  ) {
+    this.processor = processor;
+    this.semaphore = new Semaphore(concurrency);
+    this.onError = options?.onError;
+    this.onProgress = options?.onProgress;
+  }
+
+  /**
+   * Process an array of items with concurrency control
+   * Only returns successful results, errors are handled by onError callback
+   */
+  async process(items: T[]): Promise<Awaited<R>[]> {
+    const results: (Awaited<R> | undefined)[] = new Array(items.length);
+    let completed = 0;
+    const total = items.length;
+
+    const promises = items.map(async (item, index) => {
+      await this.semaphore.acquire();
+      try {
+        const result = await this.processor(item, index);
+        completed++;
+        this.onProgress?.(completed, total);
+        return result;
+      } catch (error) {
+        completed++;
+        this.onProgress?.(completed, total);
+        this.onError?.(error as Error, item, index);
+        return undefined;
+      } finally {
+        this.semaphore.release();
+      }
+    });
+
+    const allResults = await Promise.all(promises);
+    return allResults.filter((r): r is Awaited<R> => r !== undefined);
+  }
+
+  /**
+   * Process an array and return both results and errors
+   */
+  async processWithErrors(items: T[]): Promise<{
+    results: R[];
+    errors: Array<{ error: Error; item: T; index: number }>;
+  }> {
+    const results: R[] = [];
+    const errors: Array<{ error: Error; item: T; index: number }> = [];
+    let completed = 0;
+    const total = items.length;
+
+    const promises = items.map(async (item, index) => {
+      await this.semaphore.acquire();
+      try {
+        const result = await this.processor(item, index);
+        results.push(result);
+        completed++;
+        this.onProgress?.(completed, total);
+      } catch (error) {
+        completed++;
+        this.onProgress?.(completed, total);
+        errors.push({ error: error as Error, item, index });
+      } finally {
+        this.semaphore.release();
+      }
+    });
+
+    await Promise.all(promises);
+    return { results, errors };
+  }
+}