Update engage-api/login_template.txt

bun.lock

@@ -6,6 +6,7 @@
       "dependencies": {
         "axios": "^1.9.0",
         "cors": "^2.8.5",
+        "crypto": "^1.0.1",
         "dotenv": "^16.5.0",
         "express": "^5.1.0",
         "p-limit": "^6.2.0",
@@ -104,6 +105,8 @@
 
     "cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="],
 
+    "crypto": ["crypto@1.0.1", "", {}, "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="],
+
     "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],
 
     "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],

docker-compose.yaml

@@ -15,7 +15,7 @@ services:
       - cca_network
 
   redis:
-    image: "redis:7.2-alpine"
+    image: "redis:8.0-alpine"
     container_name: dsas-cca-redis
     command: redis-server --requirepass "dsas-cca"
     volumes:

engage-api/get-activity.ts

@@ -1,4 +1,4 @@
-// ./engage-api/get-activity.ts
+// engage-api/get-activity.ts
 import axios from 'axios';
 import { readFile,writeFile,unlink } from 'fs/promises';
 import { resolve } from 'path';
@@ -11,7 +11,7 @@ interface ActivityResponse {
   [key: string]: any;
 }
 
-// --- Custom Error for Authentication ---
+// Custom Error for Authentication
 class AuthenticationError extends Error {
   status: number;
 
@@ -26,7 +26,7 @@ class AuthenticationError extends Error {
 const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt');
 let _inMemoryCookie: string | null = null;
 
-// --- Cookie Cache Helper Functions ---
+// Cookie Cache Helper Functions
 async function loadCachedCookie(): Promise < string | null > {
   if (_inMemoryCookie) {
     logger.debug("Using in-memory cached cookie.");
@@ -93,10 +93,15 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
         'Cookie': cookieString,
         'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
       };
-      const payload = { "activityID": "3350" };
+      const payload = {
+        "activityID": "3350"
+      };
 
       logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`);
-      await axios.post(url, payload, { headers, timeout: 20000 });
+      await axios.post(url, payload, {
+        headers,
+        timeout: 20000
+      });
 
       logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
       return true;
@@ -117,12 +122,14 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
   return false;
 }
 
-// --- Core API Interaction Functions ---
+// Core API Interaction Functions
 async function getSessionId(): Promise < string | null > {
   const url = 'https://engage.nkcswx.cn/Login.aspx';
   try {
     const response = await axios.get(url, {
-      headers: { 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)' }
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)'
+      }
     });
     const setCookieHeader = response.headers['set-cookie'];
     if (setCookieHeader && setCookieHeader.length > 0) {
@@ -215,7 +222,9 @@ async function getActivityDetailsRaw(
     'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
     'X-Requested-With': 'XMLHttpRequest'
   };
-  const payload = { "activityID": String(activityId) };
+  const payload = {
+    "activityID": String(activityId)
+  };
 
   for (let attempt = 0; attempt < maxRetries; attempt++) {
     try {
@@ -242,6 +251,7 @@ async function getActivityDetailsRaw(
         throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
       }
       logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
+
       if (error.response) {
         logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
       }
@@ -343,4 +353,4 @@ export async function fetchActivityData(
 }
 
 // Optionally
-export { clearCookieCache, testCookieValidity };
+//export { clearCookieCache,testCookieValidity };

index.ts

@@ -24,19 +24,9 @@ import {
   cleanupOrphanedS3Images
 } from './services/cache-manager';
 import { logger } from './utils/logger';
+import type { ActivityData } from './models/activity'
 
 // Define interfaces for our data structures
-interface ActivityData {
-    id?: string;
-    name?: string;
-    photo?: string;
-    lastCheck?: string;
-    source?: string;
-    error?: string;
-    cache?: string;
-    [key: string]: any;
-}
-
 interface StaffData {
   lastCheck?: string;
   cache?: string;
@@ -124,9 +114,10 @@ async function fetchProcessAndStoreActivity(activityId: string): Promise<Process
 app.get('/', (_req: Request, res: Response) => {
   res.send('Welcome to the DSAS CCA API!<br/>\
     GET /v1/activity/list<br/>\
-        GET /v1/activity/list?category=<br/>\
+    GET /v1/activity/list?category={categoryName}<br/>\
-        GET /v1/activity/list?academicYear=<br/>\
+    GET /v1/activity/list?academicYear={YYYY/YYYY}<br/>\
-        GET /v1/activity/list?grade=<br/>\
+    GET /v1/activity/list?grade={1-12}<br/>\
+    GET /v1/activity/list?isStudentLed={true|false}<br/>\
     GET /v1/activity/category<br/>\
     GET /v1/activity/academicYear<br/>\
     GET /v1/activity/:activityId<br/>\
@@ -139,126 +130,100 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
     const category       = req.query.category as string | undefined;
     const academicYear   = req.query.academicYear as string | undefined;
     const grade          = req.query.grade as string | undefined;
+    const isStudentLedQ  = req.query.isStudentLed as string | undefined;
 
-        // Validate academicYear format if provided (YYYY/YYYY)
+    /* ---------- validate query params ---------- */
+    // academicYear (YYYY/YYYY)
     if (academicYear !== undefined) {
       const academicYearRegex = /^\d{4}\/\d{4}$/;
       if (!academicYearRegex.test(academicYear)) {
         return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' });
       }
     }
-        
+    // grade (1 – 12)
-        // Validate grade if provided
     let validGrade: number | null = null;
     if (grade !== undefined) {
       const parsedGrade = parseInt(grade, 10);
-            if (!isNaN(parsedGrade) && parsedGrade > 0 && parsedGrade <= 12) {
+      if (isNaN(parsedGrade) || parsedGrade < 1 || parsedGrade > 12) {
-                validGrade = parsedGrade;
-            } else {
         return res.status(400).json({ error: 'Invalid grade parameter. Must be a number between 1 and 12.' });
       }
+      validGrade = parsedGrade;
     }
+    // isStudentLed ("true" | "false")
+    let isStudentLedFilter: boolean | null = null;
+    if (isStudentLedQ !== undefined) {
+      if (isStudentLedQ === 'true')  isStudentLedFilter = true;
+      else if (isStudentLedQ === 'false') isStudentLedFilter = false;
+      else {
+        return res.status(400).json({ error: 'Invalid isStudentLed parameter. Must be "true" or "false".' });
+      }
+    }
+    logger.info(`Request /v1/activity/list filters: ${JSON.stringify({ category, academicYear, grade: validGrade, isStudentLed: isStudentLedFilter })}`);
 
-        logger.info(`Request received for /v1/activity/list with filters: ${JSON.stringify({category, academicYear, grade: validGrade})}`);
+    /* ---------- fetch & cache ---------- */
-        
     const activityKeys = await getAllActivityKeys();
-        const clubList: Record<string, {name: string, photo: string}> = {};
+    const clubList: Record<string, { name: string; photo: string }> = {};
 
     if (!activityKeys || activityKeys.length === 0) {
       logger.info('No activity keys found in Redis for list.');
       return res.json({});
     }
 
-        // Fetch all activity data in parallel
+    const allActivities = await Promise.all(
-        const allActivityDataPromises = activityKeys.map(async (key) => {
+      activityKeys.map(async k => getActivityData(k.substring(ACTIVITY_KEY_PREFIX.length)))
-            const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
+    );
-            return getActivityData(activityId);
-        });
 
-        const allActivities = await Promise.all(allActivityDataPromises);
+    /* ---------- gather available filter values for validation ---------- */
-        
-        // First pass: collect all available categories for validation
     const availableCategories    = new Set<string>();
     const availableAcademicYears = new Set<string>();
 
-        allActivities.forEach((activityData: ActivityData | null) => {
+    allActivities.forEach(a => {
-            if (activityData && 
+      if (a && !a.error && a.source !== 'api-fetch-empty') {
-                !activityData.error && 
+        if (a.category)      availableCategories.add(a.category);
-                activityData.source !== 'api-fetch-empty') {
+        if (a.academicYear)  availableAcademicYears.add(a.academicYear);
-                if (activityData.category) {
-                    availableCategories.add(activityData.category);
-                }
-                if (activityData.academicYear) {
-                    availableAcademicYears.add(activityData.academicYear);
-                }
       }
     });
 
-        // Validate category against available categories
     if (category && !availableCategories.has(category)) {
-            return res.status(400).json({ 
+      return res.status(400).json({ error: 'Invalid category parameter. Category not found.', availableCategories: [...availableCategories] });
-                error: 'Invalid category parameter. Category not found.',
-                availableCategories: Array.from(availableCategories)
-            });
     }
-        
-        // Validate academicYear against available years
     if (academicYear && !availableAcademicYears.has(academicYear)) {
-            return res.status(400).json({ 
+      return res.status(400).json({ error: 'Invalid academicYear parameter. Academic year not found.', availableAcademicYears: [...availableAcademicYears] });
-                error: 'Invalid academicYear parameter. Academic year not found.',
-                availableAcademicYears: Array.from(availableAcademicYears)
-            });
     }
 
-        // Apply filters and collect club data
+    /* ---------- apply filters ---------- */
-        allActivities.forEach((activityData: ActivityData | null) => {
+    allActivities.forEach(a => {
-            if (activityData &&
+      if (
-                activityData.id &&
+        a &&
-                activityData.name &&
+        a.id &&
-                !activityData.error &&
+        a.name &&
-                activityData.source !== 'api-fetch-empty') {
+        !a.error &&
-                
+        a.source !== 'api-fetch-empty'
-                // Check if it matches category filter if provided
+      ) {
-                if (category && activityData.category !== category) {
+        // category
-                    return; // Skip this activity
+        if (category && a.category !== category) return;
-                }
+        // academicYear
-                
+        if (academicYear && a.academicYear !== academicYear) return;
-                // Check if it matches academicYear filter if provided
+        // grade
-                if (academicYear && activityData.academicYear !== academicYear) {
-                    return; // Skip this activity
-                }
-                
-                // Check if it matches grade filter if provided
         if (validGrade !== null) {
-                    // Skip if grades are null
+          if (!a.grades || a.grades.min == null || a.grades.max == null) return;
-                    if (!activityData.grades || 
+          const minG = parseInt(a.grades.min, 10);
-                        activityData.grades.min === null || 
+          const maxG = parseInt(a.grades.max, 10);
-                        activityData.grades.max === null) {
+          if (isNaN(minG) || isNaN(maxG) || validGrade < minG || validGrade > maxG) return;
-                        return; // Skip this activity
         }
-                    
+        // isStudentLed
-                    const minGrade = parseInt(activityData.grades.min, 10);
+        if (isStudentLedFilter !== null) {
-                    const maxGrade = parseInt(activityData.grades.max, 10);
+          // Treat missing value as false
-                    
+          const led = a.isStudentLed ?? false;
-                    // Skip if grade is out of range or if parsing fails
+          if (led !== isStudentLedFilter) return;
-                    if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) {
-                        return; // Skip this activity
         }
-                }
+        clubList[a.id] = { name: a.name, photo: a.photo || '' };
-                
-                // Add to result object with name and photo
-                clubList[activityData.id] = {
-                    name: activityData.name,
-                    photo: activityData.photo || ""
-                };
       }
     });
-
+    logger.info(`Returning ${Object.keys(clubList).length} clubs after filtering.`);
-        logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`);
     res.json(clubList);
-
+  } catch (err) {
-    } catch (error) {
+    logger.error('Error in /v1/activity/list endpoint:', err);
-        logger.error('Error in /v1/activity/list endpoint:', error);
     res.status(500).json({ error: 'An internal server error occurred while generating activity list.' });
   }
 });
@@ -274,7 +239,6 @@ app.get('/v1/activity/category', async (_req: Request, res: Response) => {
       logger.info('No activity keys found in Redis for categories.');
       return res.json({});
     }
-
     // Fetch all activity data in parallel
     const allActivityDataPromises = activityKeys.map(async (key) => {
       const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
@@ -298,7 +262,6 @@ app.get('/v1/activity/category', async (_req: Request, res: Response) => {
 
     logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`);
     res.json(categoryMap);
-
   } catch (error) {
     logger.error('Error in /v1/activity/category endpoint:', error);
     res.status(500).json({ error: 'An internal server error occurred while generating category list.' });
@@ -312,38 +275,47 @@ app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {
     const activityKeys = await getAllActivityKeys();
     const academicYearMap: Record<string, number> = {};
 
-        if (!activityKeys || activityKeys.length === 0) {
+    if (!activityKeys?.length) {
       logger.info('No activity keys found in Redis for academic years.');
       return res.json({});
     }
-
+    // 1. Fetch all activity data in parallel
-        // Fetch all activity data in parallel
+    const allActivities = await Promise.all(
-        const allActivityDataPromises = activityKeys.map(async (key) => {
+      activityKeys.map(async (key) => {
         const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
         return getActivityData(activityId);
-        });
+      })
-
+    );
-        const allActivities = await Promise.all(allActivityDataPromises);
+    // 2. Count activities per academic year
-
     allActivities.forEach((activityData: ActivityData | null) => {
-            if (activityData && 
+      if (
+        activityData &&
         activityData.academicYear &&
         !activityData.error &&
-                activityData.source !== 'api-fetch-empty') {
+        activityData.source !== 'api-fetch-empty'
-                if (academicYearMap[activityData.academicYear]) {
+      ) {
-                    academicYearMap[activityData.academicYear] = (academicYearMap[activityData.academicYear] ?? 0) + 1;
+        academicYearMap[activityData.academicYear] =
-                } else {
+          (academicYearMap[activityData.academicYear] ?? 0) + 1;
-                    academicYearMap[activityData.academicYear] = 1;
-                }
       }
     });
-
+    // 3. Sort the years in descending order (based on the start year)
-        logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`);
+    const sortedAcademicYearMap: Record<string, number> = Object.fromEntries(
-        res.json(academicYearMap);
+      Object.entries(academicYearMap).sort(([yearA], [yearB]) => {
-
+        const startA = parseInt(yearA.split('/')[0], 10);
+        const startB = parseInt(yearB.split('/')[0], 10);
+        return startB - startA;
+      })
+    );
+    logger.info(
+      `Returning list of ${Object.keys(sortedAcademicYearMap).length} academic years.`
+    );
+    res.json(sortedAcademicYearMap);
   } catch (error) {
     logger.error('Error in /v1/activity/academicYear endpoint:', error);
-        res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' });
+    res.status(500).json({
+      error:
+        'An internal server error occurred while generating academic year list.',
+    });
   }
 });
 
@@ -379,7 +351,6 @@ app.get('/v1/activity/:activityId', async (req: Request, res: Response) => {
       return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
     }
     res.status(status).json(liveActivity);
-
   } catch (error) {
     logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
     res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
@@ -440,7 +411,7 @@ async function performBackgroundTasks(): Promise<void> {
   }
 }
 
-// --- Start Server and Background Tasks ---
+// Start Server and Background Tasks
 async function startServer(): Promise<void> {
   const redis = getRedisClient();
   if (!redis) {

models/activity.ts

@@ -1,10 +1,10 @@
 // src/models/activity.ts
 export interface ActivityData {
   // Include all common properties
-  id?: string | null | undefined;
+  id?: string | null;
   name?: string | null;
   description?: string | null;
-  photo?: string | null | undefined;
+  photo?: string | null;
   academicYear?: string | null;
   category?: string | null;
   isPreSignup?: boolean | null;

package.json

@@ -14,6 +14,7 @@
   "dependencies": {
     "axios": "^1.9.0",
     "cors": "^2.8.5",
+    "crypto": "^1.0.1",
     "dotenv": "^16.5.0",
     "express": "^5.1.0",
     "p-limit": "^6.2.0",

services/cache-manager.ts

@@ -77,7 +77,6 @@ async function processAndCacheActivity(activityId: string): Promise<ActivityData
         }
       }
     }
-
     structuredActivity.lastCheck = new Date().toISOString();
     await setActivityData(activityId, structuredActivity);
     return structuredActivity;

services/s3-service.ts

@@ -3,6 +3,7 @@ import { S3Client } from "bun";
 import { v4 as uuidv4 } from 'uuid';
 import { config } from 'dotenv';
 import sharp from 'sharp';
+import crypto from 'crypto';
 import { logger } from '../utils/logger';
 import { decodeBase64Image } from '../utils/image-processor';
 
@@ -38,9 +39,10 @@ if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {
 
 /**
  * Uploads an image from a base64 string to S3, converting it to AVIF format.
+ * Uses MD5 checksum as filename and checks for duplicates before uploading.
  * @param base64Data - The base64 content (without the data URI prefix)
  * @param originalFormat - The image format (e.g., 'png', 'jpeg')
- * @param activityId - The activity ID, used for naming
+ * @param activityId - The activity ID, used for logging purposes
  * @returns The public URL of the uploaded image or null on error
  */
 export async function uploadImageFromBase64(
@@ -56,11 +58,9 @@ export async function uploadImageFromBase64(
     logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
     return null;
   }
-
   try {
     // First decode the base64 image
     const imageBuffer = decodeBase64Image(base64Data);
-        
     // Convert to AVIF format with quality 80 using Sharp
     const avifBuffer = await sharp(imageBuffer)
       .avif({ 
@@ -70,20 +70,26 @@ export async function uploadImageFromBase64(
         // effort: 4,
       })
       .toBuffer();
-        
+    // Calculate MD5 checksum of the converted AVIF image
-        // Use .avif extension for the object key
+    const md5Hash = crypto.createHash('md5').update(avifBuffer).digest('hex');
-        const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`;
+    const objectKey = `${PUBLIC_URL_FILE_PREFIX}/${md5Hash}.avif`;
-        
+    // Check if file with this checksum already exists
-        // Using Bun's S3Client file API
     const s3File = s3Client.file(objectKey);
+    const exists = await s3File.exists();
     
+    if (exists) {
+      const publicUrl = constructS3Url(objectKey);
+      logger.info(`Image already exists in S3 (MD5: ${md5Hash}), returning existing URL: ${publicUrl}`);
+      return publicUrl;
+    }
+    // File doesn't exist, proceed with upload
     await s3File.write(avifBuffer, {
       type: 'image/avif',
       acl: 'public-read'
     });
     
     const publicUrl = constructS3Url(objectKey);
-        logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`);
+    logger.info(`Image uploaded to S3 as AVIF (MD5: ${md5Hash}): ${publicUrl}`);
     return publicUrl;
   } catch (error) {
     logger.error(`S3 Upload Error for activity ${activityId}:`, error);
@@ -116,7 +122,6 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
         startAfter,
         maxKeys: 1000
       });
-            
       if (result.contents) {
         // Add keys to our array, filtering out "directories"
         result.contents.forEach(item => {
@@ -124,21 +129,17 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
             objectKeys.push(item.key);
           }
         });
-                
         // Get the last key for pagination
         if (result.contents?.length > 0) {
           startAfter = result.contents[result.contents.length - 1]?.key;
         }
       }
-            
       isTruncated = result.isTruncated || false;
-            
       // Safety check to prevent infinite loops
       if (result.contents?.length === 0) {
         break;
       }
     }
-        
     logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
     return objectKeys;
   } catch (error) {
@@ -161,7 +162,6 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
     logger.info('No objects to delete from S3.');
     return true;
   }
-
   try {
     // With Bun's S3Client, we need to delete objects one by one
     // Process in batches of 100 for better performance
@@ -171,12 +171,10 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
     
     for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) {
       const batch = objectKeysArray.slice(i, i + BATCH_SIZE);
-            
       // Process batch in parallel
       const results = await Promise.allSettled(
         batch.map(key => s3Client!.delete(key))
       );
-            
       // Count successes and failures
       for (const result of results) {
         if (result.status === 'fulfilled') {
@@ -187,7 +185,6 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
         }
       }
     }
-        
     logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
     return errorCount === 0; // True if all succeeded
   } catch (error) {
@@ -205,7 +202,6 @@ export function constructS3Url(objectKey: string): string {
   if (!S3_ENDPOINT || !BUCKET_NAME) {
     return '';
   }
-    
   // Ensure S3_ENDPOINT does not end with a slash
   const s3Base = S3_ENDPOINT.replace(/\/$/, '');
   // Ensure BUCKET_NAME does not start or end with a slash