feat: skip duplicate image on remote

bun.lock

@@ -6,6 +6,7 @@
       "dependencies": {
         "axios": "^1.9.0",
         "cors": "^2.8.5",
+        "crypto": "^1.0.1",
         "dotenv": "^16.5.0",
         "express": "^5.1.0",
         "p-limit": "^6.2.0",
@@ -104,6 +105,8 @@
 
     "cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="],
 
+    "crypto": ["crypto@1.0.1", "", {}, "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="],
+
     "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],
 
     "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],

engage-api/get-activity.ts

@@ -1,21 +1,21 @@
-// ./engage-api/get-activity.ts
+// engage-api/get-activity.ts
 import axios from 'axios';
-import { readFile, writeFile, unlink } from 'fs/promises';
+import { readFile,writeFile,unlink } from 'fs/promises';
 import { resolve } from 'path';
 import { logger } from '../utils/logger';
 
 // Define interfaces for our data structures
 interface ActivityResponse {
   d: string;
-  isError?: boolean;
+  isError ? : boolean;
   [key: string]: any;
 }
 
-// --- Custom Error for Authentication ---
+// Custom Error for Authentication
 class AuthenticationError extends Error {
   status: number;
 
-  constructor(message: string = "Authentication failed, cookie may be invalid.", status?: number) {
+  constructor(message: string = "Authentication failed, cookie may be invalid.", status ? : number) {
     super(message);
     this.name = "AuthenticationError";
     this.status = status || 0;
@@ -26,8 +26,8 @@ class AuthenticationError extends Error {
 const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt');
 let _inMemoryCookie: string | null = null;
 
-// --- Cookie Cache Helper Functions ---
+// Cookie Cache Helper Functions
-async function loadCachedCookie(): Promise<string | null> {
+async function loadCachedCookie(): Promise < string | null > {
   if (_inMemoryCookie) {
     logger.debug("Using in-memory cached cookie.");
     return _inMemoryCookie;
@@ -49,7 +49,7 @@ async function loadCachedCookie(): Promise<string | null> {
   return null;
 }
 
-async function saveCookieToCache(cookieString: string): Promise<void> {
+async function saveCookieToCache(cookieString: string): Promise < void > {
   if (!cookieString) {
     logger.warn("Attempted to save an empty or null cookie. Aborting save.");
     return;
@@ -63,7 +63,7 @@ async function saveCookieToCache(cookieString: string): Promise<void> {
   }
 }
 
-async function clearCookieCache(): Promise<void> {
+async function clearCookieCache(): Promise < void > {
   _inMemoryCookie = null;
   try {
     await unlink(COOKIE_FILE_PATH);
@@ -77,7 +77,7 @@ async function clearCookieCache(): Promise<void> {
   }
 }
 
-async function testCookieValidity(cookieString: string): Promise<boolean> {
+async function testCookieValidity(cookieString: string): Promise < boolean > {
   if (!cookieString) return false;
   logger.debug("Testing cookie validity...");
 
@@ -93,10 +93,15 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
         'Cookie': cookieString,
         'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
       };
-      const payload = { "activityID": "3350" };
+      const payload = {
+        "activityID": "3350"
+      };
 
       logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`);
-      await axios.post(url, payload, { headers, timeout: 20000 });
+      await axios.post(url, payload, {
+        headers,
+        timeout: 20000
+      });
 
       logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
       return true;
@@ -117,12 +122,14 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
   return false;
 }
 
-// --- Core API Interaction Functions ---
+// Core API Interaction Functions
-async function getSessionId(): Promise<string | null> {
+async function getSessionId(): Promise < string | null > {
   const url = 'https://engage.nkcswx.cn/Login.aspx';
   try {
     const response = await axios.get(url, {
-      headers: { 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)' }
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)'
+      }
     });
     const setCookieHeader = response.headers['set-cookie'];
     if (setCookieHeader && setCookieHeader.length > 0) {
@@ -141,7 +148,7 @@ async function getSessionId(): Promise<string | null> {
   }
 }
 
-async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise<string | null> {
+async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise < string | null > {
   const url = 'https://engage.nkcswx.cn/Login.aspx';
   try {
     let templateData = await readFile(templateFilePath, 'utf8');
@@ -191,7 +198,7 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
   }
 }
 
-async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise<string> {
+async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise < string > {
   logger.debug('Attempting to get complete cookie string (login process).');
   const sessionId = await getSessionId();
   if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId.");
@@ -207,7 +214,7 @@ async function getActivityDetailsRaw(
   cookies: string,
   maxRetries: number = 3,
   timeoutMilliseconds: number = 20000
-): Promise<string | null> {
+): Promise < string | null > {
   const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
   const headers = {
     'Content-Type': 'application/json; charset=UTF-8',
@@ -215,7 +222,9 @@ async function getActivityDetailsRaw(
     'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
     'X-Requested-With': 'XMLHttpRequest'
   };
-  const payload = { "activityID": String(activityId) };
+  const payload = {
+    "activityID": String(activityId)
+  };
 
   for (let attempt = 0; attempt < maxRetries; attempt++) {
     try {
@@ -242,6 +251,7 @@ async function getActivityDetailsRaw(
         throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
       }
       logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
+
       if (error.response) {
         logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
       }
@@ -270,7 +280,7 @@ export async function fetchActivityData(
   userPwd: string,
   templateFileName: string = "login_template.txt",
   forceLogin: boolean = false
-): Promise<any | null> {
+): Promise < any | null > {
   let currentCookie = forceLogin ? null : await loadCachedCookie();
 
   if (forceLogin && currentCookie) {
@@ -343,4 +353,4 @@ export async function fetchActivityData(
 }
 
 // Optionally
-export { clearCookieCache, testCookieValidity };
+//export { clearCookieCache,testCookieValidity };

index.ts

@@ -24,19 +24,9 @@ import {
   cleanupOrphanedS3Images
 } from './services/cache-manager';
 import { logger } from './utils/logger';
+import type { ActivityData } from './models/activity'
 
 // Define interfaces for our data structures
-interface ActivityData {
-    id?: string;
-    name?: string;
-    photo?: string;
-    lastCheck?: string;
-    source?: string;
-    error?: string;
-    cache?: string;
-    [key: string]: any;
-}
-
 interface StaffData {
   lastCheck?: string;
   cache?: string;
@@ -139,7 +129,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
     const category = req.query.category as string | undefined;
     const academicYear = req.query.academicYear as string | undefined;
     const grade = req.query.grade as string | undefined;
-        
     // Validate academicYear format if provided (YYYY/YYYY)
     if (academicYear !== undefined) {
       const academicYearRegex = /^\d{4}\/\d{4}$/;
@@ -147,7 +136,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
         return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' });
       }
     }
-        
     // Validate grade if provided
     let validGrade: number | null = null;
     if (grade !== undefined) {
@@ -168,7 +156,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
       logger.info('No activity keys found in Redis for list.');
       return res.json({});
     }
-
     // Fetch all activity data in parallel
     const allActivityDataPromises = activityKeys.map(async (key) => {
       const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
@@ -176,7 +163,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
     });
 
     const allActivities = await Promise.all(allActivityDataPromises);
-        
     // First pass: collect all available categories for validation
     const availableCategories = new Set<string>();
     const availableAcademicYears = new Set<string>();
@@ -193,7 +179,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
         }
       }
     });
-        
     // Validate category against available categories
     if (category && !availableCategories.has(category)) {
       return res.status(400).json({ 
@@ -201,7 +186,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
         availableCategories: Array.from(availableCategories)
       });
     }
-        
     // Validate academicYear against available years
     if (academicYear && !availableAcademicYears.has(academicYear)) {
       return res.status(400).json({ 
@@ -209,7 +193,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
         availableAcademicYears: Array.from(availableAcademicYears)
       });
     }
-        
     // Apply filters and collect club data
     allActivities.forEach((activityData: ActivityData | null) => {
       if (activityData &&
@@ -217,17 +200,14 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
         activityData.name &&
         !activityData.error &&
         activityData.source !== 'api-fetch-empty') {
-                
         // Check if it matches category filter if provided
         if (category && activityData.category !== category) {
           return; // Skip this activity
         }
-                
         // Check if it matches academicYear filter if provided
         if (academicYear && activityData.academicYear !== academicYear) {
           return; // Skip this activity
         }
-                
         // Check if it matches grade filter if provided
         if (validGrade !== null) {
           // Skip if grades are null
@@ -239,13 +219,11 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
           
           const minGrade = parseInt(activityData.grades.min, 10);
           const maxGrade = parseInt(activityData.grades.max, 10);
-                    
           // Skip if grade is out of range or if parsing fails
           if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) {
             return; // Skip this activity
           }
         }
-                
         // Add to result object with name and photo
         clubList[activityData.id] = {
           name: activityData.name,
@@ -253,7 +231,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
         };
       }
     });
-
     logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`);
     res.json(clubList);
 
@@ -274,7 +251,6 @@ app.get('/v1/activity/category', async (_req: Request, res: Response) => {
       logger.info('No activity keys found in Redis for categories.');
       return res.json({});
     }
-
     // Fetch all activity data in parallel
     const allActivityDataPromises = activityKeys.map(async (key) => {
       const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
@@ -298,7 +274,6 @@ app.get('/v1/activity/category', async (_req: Request, res: Response) => {
 
     logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`);
     res.json(categoryMap);
-
   } catch (error) {
     logger.error('Error in /v1/activity/category endpoint:', error);
     res.status(500).json({ error: 'An internal server error occurred while generating category list.' });
@@ -316,7 +291,6 @@ app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {
       logger.info('No activity keys found in Redis for academic years.');
       return res.json({});
     }
-
     // Fetch all activity data in parallel
     const allActivityDataPromises = activityKeys.map(async (key) => {
       const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
@@ -340,7 +314,6 @@ app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {
 
     logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`);
     res.json(academicYearMap);
-
   } catch (error) {
     logger.error('Error in /v1/activity/academicYear endpoint:', error);
     res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' });
@@ -379,7 +352,6 @@ app.get('/v1/activity/:activityId', async (req: Request, res: Response) => {
       return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
     }
     res.status(status).json(liveActivity);
-
   } catch (error) {
     logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
     res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
@@ -440,7 +412,7 @@ async function performBackgroundTasks(): Promise<void> {
   }
 }
 
-// --- Start Server and Background Tasks ---
+// Start Server and Background Tasks
 async function startServer(): Promise<void> {
   const redis = getRedisClient();
   if (!redis) {

models/activity.ts

@@ -1,10 +1,10 @@
 // src/models/activity.ts
 export interface ActivityData {
   // Include all common properties
-  id?: string | null | undefined;
+  id?: string | null;
   name?: string | null;
   description?: string | null;
-  photo?: string | null | undefined;
+  photo?: string | null;
   academicYear?: string | null;
   category?: string | null;
   isPreSignup?: boolean | null;

package.json

@@ -14,6 +14,7 @@
   "dependencies": {
     "axios": "^1.9.0",
     "cors": "^2.8.5",
+    "crypto": "^1.0.1",
     "dotenv": "^16.5.0",
     "express": "^5.1.0",
     "p-limit": "^6.2.0",

services/cache-manager.ts

@@ -77,7 +77,6 @@ async function processAndCacheActivity(activityId: string): Promise<ActivityData
         }
       }
     }
-
     structuredActivity.lastCheck = new Date().toISOString();
     await setActivityData(activityId, structuredActivity);
     return structuredActivity;

services/s3-service.ts

@@ -3,6 +3,7 @@ import { S3Client } from "bun";
 import { v4 as uuidv4 } from 'uuid';
 import { config } from 'dotenv';
 import sharp from 'sharp';
+import crypto from 'crypto';
 import { logger } from '../utils/logger';
 import { decodeBase64Image } from '../utils/image-processor';
 
@@ -38,9 +39,10 @@ if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {
 
 /**
  * Uploads an image from a base64 string to S3, converting it to AVIF format.
+ * Uses MD5 checksum as filename and checks for duplicates before uploading.
  * @param base64Data - The base64 content (without the data URI prefix)
  * @param originalFormat - The image format (e.g., 'png', 'jpeg')
- * @param activityId - The activity ID, used for naming
+ * @param activityId - The activity ID, used for logging purposes
  * @returns The public URL of the uploaded image or null on error
  */
 export async function uploadImageFromBase64(
@@ -56,11 +58,9 @@ export async function uploadImageFromBase64(
     logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
     return null;
   }
-
   try {
     // First decode the base64 image
     const imageBuffer = decodeBase64Image(base64Data);
-        
     // Convert to AVIF format with quality 80 using Sharp
     const avifBuffer = await sharp(imageBuffer)
       .avif({ 
@@ -70,20 +70,26 @@ export async function uploadImageFromBase64(
         // effort: 4,
       })
       .toBuffer();
-        
+    // Calculate MD5 checksum of the converted AVIF image
-        // Use .avif extension for the object key
+    const md5Hash = crypto.createHash('md5').update(avifBuffer).digest('hex');
-        const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`;
+    const objectKey = `${PUBLIC_URL_FILE_PREFIX}/${md5Hash}.avif`;
-        
+    // Check if file with this checksum already exists
-        // Using Bun's S3Client file API
     const s3File = s3Client.file(objectKey);
+    const exists = await s3File.exists();
     
+    if (exists) {
+      const publicUrl = constructS3Url(objectKey);
+      logger.info(`Image already exists in S3 (MD5: ${md5Hash}), returning existing URL: ${publicUrl}`);
+      return publicUrl;
+    }
+    // File doesn't exist, proceed with upload
     await s3File.write(avifBuffer, {
       type: 'image/avif',
       acl: 'public-read'
     });
     
     const publicUrl = constructS3Url(objectKey);
-        logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`);
+    logger.info(`Image uploaded to S3 as AVIF (MD5: ${md5Hash}): ${publicUrl}`);
     return publicUrl;
   } catch (error) {
     logger.error(`S3 Upload Error for activity ${activityId}:`, error);
@@ -116,7 +122,6 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
         startAfter,
         maxKeys: 1000
       });
-            
       if (result.contents) {
         // Add keys to our array, filtering out "directories"
         result.contents.forEach(item => {
@@ -124,21 +129,17 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
             objectKeys.push(item.key);
           }
         });
-                
         // Get the last key for pagination
         if (result.contents?.length > 0) {
           startAfter = result.contents[result.contents.length - 1]?.key;
         }
       }
-            
       isTruncated = result.isTruncated || false;
-            
       // Safety check to prevent infinite loops
       if (result.contents?.length === 0) {
         break;
       }
     }
-        
     logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
     return objectKeys;
   } catch (error) {
@@ -161,7 +162,6 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
     logger.info('No objects to delete from S3.');
     return true;
   }
-
   try {
     // With Bun's S3Client, we need to delete objects one by one
     // Process in batches of 100 for better performance
@@ -171,12 +171,10 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
     
     for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) {
       const batch = objectKeysArray.slice(i, i + BATCH_SIZE);
-            
       // Process batch in parallel
       const results = await Promise.allSettled(
         batch.map(key => s3Client!.delete(key))
       );
-            
       // Count successes and failures
       for (const result of results) {
         if (result.status === 'fulfilled') {
@@ -187,7 +185,6 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
         }
       }
     }
-        
     logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
     return errorCount === 0; // True if all succeeded
   } catch (error) {
@@ -205,7 +202,6 @@ export function constructS3Url(objectKey: string): string {
   if (!S3_ENDPOINT || !BUCKET_NAME) {
     return '';
   }
-    
   // Ensure S3_ENDPOINT does not end with a slash
   const s3Base = S3_ENDPOINT.replace(/\/$/, '');
   // Ensure BUCKET_NAME does not start or end with a slash