feat: skip duplicate image on remote

improve: code structure
2025-05-12 23:46:25 -04:00 · 2025-05-12 21:45:57 -04:00
10 changed files with 859 additions and 878 deletions
--- a/bun.lock
+++ b/bun.lock
@@ -6,6 +6,7 @@
      "dependencies": {
        "axios": "^1.9.0",
        "cors": "^2.8.5",
+        "crypto": "^1.0.1",
        "dotenv": "^16.5.0",
        "express": "^5.1.0",
        "p-limit": "^6.2.0",
@@ -104,6 +105,8 @@

    "cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="],

+    "crypto": ["crypto@1.0.1", "", {}, "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="],
+
    "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],

    "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
--- a/engage-api/get-activity.ts
+++ b/engage-api/get-activity.ts
@@ -1,21 +1,21 @@
-// ./engage-api/get-activity.ts
+// engage-api/get-activity.ts
 import axios from 'axios';
-import { readFile, writeFile, unlink } from 'fs/promises';
+import { readFile,writeFile,unlink } from 'fs/promises';
 import { resolve } from 'path';
 import { logger } from '../utils/logger';

 // Define interfaces for our data structures
 interface ActivityResponse {
  d: string;
-  isError?: boolean;
+  isError ? : boolean;
  [key: string]: any;
 }

-// --- Custom Error for Authentication ---
+// Custom Error for Authentication
 class AuthenticationError extends Error {
  status: number;

-  constructor(message: string = "Authentication failed, cookie may be invalid.", status?: number) {
+  constructor(message: string = "Authentication failed, cookie may be invalid.", status ? : number) {
    super(message);
    this.name = "AuthenticationError";
    this.status = status || 0;
@@ -26,8 +26,8 @@ class AuthenticationError extends Error {
 const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt');
 let _inMemoryCookie: string | null = null;

-// --- Cookie Cache Helper Functions ---
-async function loadCachedCookie(): Promise<string | null> {
+// Cookie Cache Helper Functions
+async function loadCachedCookie(): Promise < string | null > {
  if (_inMemoryCookie) {
    logger.debug("Using in-memory cached cookie.");
    return _inMemoryCookie;
@@ -49,7 +49,7 @@ async function loadCachedCookie(): Promise<string | null> {
  return null;
 }

-async function saveCookieToCache(cookieString: string): Promise<void> {
+async function saveCookieToCache(cookieString: string): Promise < void > {
  if (!cookieString) {
    logger.warn("Attempted to save an empty or null cookie. Aborting save.");
    return;
@@ -63,7 +63,7 @@ async function saveCookieToCache(cookieString: string): Promise<void> {
  }
 }

-async function clearCookieCache(): Promise<void> {
+async function clearCookieCache(): Promise < void > {
  _inMemoryCookie = null;
  try {
    await unlink(COOKIE_FILE_PATH);
@@ -77,7 +77,7 @@ async function clearCookieCache(): Promise<void> {
  }
 }

-async function testCookieValidity(cookieString: string): Promise<boolean> {
+async function testCookieValidity(cookieString: string): Promise < boolean > {
  if (!cookieString) return false;
  logger.debug("Testing cookie validity...");

@@ -93,10 +93,15 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
        'Cookie': cookieString,
        'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
      };
-      const payload = { "activityID": "3350" };
+      const payload = {
+        "activityID": "3350"
+      };

      logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`);
-      await axios.post(url, payload, { headers, timeout: 20000 });
+      await axios.post(url, payload, {
+        headers,
+        timeout: 20000
+      });

      logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
      return true;
@@ -117,12 +122,14 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
  return false;
 }

-// --- Core API Interaction Functions ---
-async function getSessionId(): Promise<string | null> {
+// Core API Interaction Functions
+async function getSessionId(): Promise < string | null > {
  const url = 'https://engage.nkcswx.cn/Login.aspx';
  try {
    const response = await axios.get(url, {
-      headers: { 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)' }
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)'
+      }
    });
    const setCookieHeader = response.headers['set-cookie'];
    if (setCookieHeader && setCookieHeader.length > 0) {
@@ -141,7 +148,7 @@ async function getSessionId(): Promise<string | null> {
  }
 }

-async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise<string | null> {
+async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise < string | null > {
  const url = 'https://engage.nkcswx.cn/Login.aspx';
  try {
    let templateData = await readFile(templateFilePath, 'utf8');
@@ -191,7 +198,7 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
  }
 }

-async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise<string> {
+async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise < string > {
  logger.debug('Attempting to get complete cookie string (login process).');
  const sessionId = await getSessionId();
  if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId.");
@@ -207,7 +214,7 @@ async function getActivityDetailsRaw(
  cookies: string,
  maxRetries: number = 3,
  timeoutMilliseconds: number = 20000
-): Promise<string | null> {
+): Promise < string | null > {
  const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
  const headers = {
    'Content-Type': 'application/json; charset=UTF-8',
@@ -215,7 +222,9 @@ async function getActivityDetailsRaw(
    'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
    'X-Requested-With': 'XMLHttpRequest'
  };
-  const payload = { "activityID": String(activityId) };
+  const payload = {
+    "activityID": String(activityId)
+  };

  for (let attempt = 0; attempt < maxRetries; attempt++) {
    try {
@@ -242,6 +251,7 @@ async function getActivityDetailsRaw(
        throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
      }
      logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
+
      if (error.response) {
        logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
      }
@@ -270,7 +280,7 @@ export async function fetchActivityData(
  userPwd: string,
  templateFileName: string = "login_template.txt",
  forceLogin: boolean = false
-): Promise<any | null> {
+): Promise < any | null > {
  let currentCookie = forceLogin ? null : await loadCachedCookie();

  if (forceLogin && currentCookie) {
@@ -343,4 +353,4 @@ export async function fetchActivityData(
 }

 // Optionally
-export { clearCookieCache, testCookieValidity };
+//export { clearCookieCache,testCookieValidity };
--- a/index.ts
+++ b/index.ts
@@ -24,19 +24,9 @@ import {
  cleanupOrphanedS3Images
 } from './services/cache-manager';
 import { logger } from './utils/logger';
+import type { ActivityData } from './models/activity'

 // Define interfaces for our data structures
-interface ActivityData {
-    id?: string;
-    name?: string;
-    photo?: string;
-    lastCheck?: string;
-    source?: string;
-    error?: string;
-    cache?: string;
-    [key: string]: any;
-}
-
 interface StaffData {
  lastCheck?: string;
  cache?: string;
@@ -139,7 +129,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
    const category = req.query.category as string | undefined;
    const academicYear = req.query.academicYear as string | undefined;
    const grade = req.query.grade as string | undefined;
-        
    // Validate academicYear format if provided (YYYY/YYYY)
    if (academicYear !== undefined) {
      const academicYearRegex = /^\d{4}\/\d{4}$/;
@@ -147,7 +136,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
        return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' });
      }
    }
-        
    // Validate grade if provided
    let validGrade: number | null = null;
    if (grade !== undefined) {
@@ -168,7 +156,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
      logger.info('No activity keys found in Redis for list.');
      return res.json({});
    }
-
    // Fetch all activity data in parallel
    const allActivityDataPromises = activityKeys.map(async (key) => {
      const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
@@ -176,7 +163,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
    });

    const allActivities = await Promise.all(allActivityDataPromises);
-        
    // First pass: collect all available categories for validation
    const availableCategories = new Set<string>();
    const availableAcademicYears = new Set<string>();
@@ -193,7 +179,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
        }
      }
    });
-        
    // Validate category against available categories
    if (category && !availableCategories.has(category)) {
      return res.status(400).json({ 
@@ -201,7 +186,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
        availableCategories: Array.from(availableCategories)
      });
    }
-        
    // Validate academicYear against available years
    if (academicYear && !availableAcademicYears.has(academicYear)) {
      return res.status(400).json({ 
@@ -209,7 +193,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
        availableAcademicYears: Array.from(availableAcademicYears)
      });
    }
-        
    // Apply filters and collect club data
    allActivities.forEach((activityData: ActivityData | null) => {
      if (activityData &&
@@ -217,17 +200,14 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
        activityData.name &&
        !activityData.error &&
        activityData.source !== 'api-fetch-empty') {
-                
        // Check if it matches category filter if provided
        if (category && activityData.category !== category) {
          return; // Skip this activity
        }
-                
        // Check if it matches academicYear filter if provided
        if (academicYear && activityData.academicYear !== academicYear) {
          return; // Skip this activity
        }
-                
        // Check if it matches grade filter if provided
        if (validGrade !== null) {
          // Skip if grades are null
@@ -239,13 +219,11 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
          
          const minGrade = parseInt(activityData.grades.min, 10);
          const maxGrade = parseInt(activityData.grades.max, 10);
-                    
          // Skip if grade is out of range or if parsing fails
          if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) {
            return; // Skip this activity
          }
        }
-                
        // Add to result object with name and photo
        clubList[activityData.id] = {
          name: activityData.name,
@@ -253,7 +231,6 @@ app.get('/v1/activity/list', async (req: Request, res: Response) => {
        };
      }
    });
-
    logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`);
    res.json(clubList);

@@ -274,7 +251,6 @@ app.get('/v1/activity/category', async (_req: Request, res: Response) => {
      logger.info('No activity keys found in Redis for categories.');
      return res.json({});
    }
-
    // Fetch all activity data in parallel
    const allActivityDataPromises = activityKeys.map(async (key) => {
      const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
@@ -298,7 +274,6 @@ app.get('/v1/activity/category', async (_req: Request, res: Response) => {

    logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`);
    res.json(categoryMap);
-
  } catch (error) {
    logger.error('Error in /v1/activity/category endpoint:', error);
    res.status(500).json({ error: 'An internal server error occurred while generating category list.' });
@@ -316,7 +291,6 @@ app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {
      logger.info('No activity keys found in Redis for academic years.');
      return res.json({});
    }
-
    // Fetch all activity data in parallel
    const allActivityDataPromises = activityKeys.map(async (key) => {
      const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
@@ -340,7 +314,6 @@ app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {

    logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`);
    res.json(academicYearMap);
-
  } catch (error) {
    logger.error('Error in /v1/activity/academicYear endpoint:', error);
    res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' });
@@ -379,7 +352,6 @@ app.get('/v1/activity/:activityId', async (req: Request, res: Response) => {
      return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
    }
    res.status(status).json(liveActivity);
-
  } catch (error) {
    logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
    res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
@@ -440,7 +412,7 @@ async function performBackgroundTasks(): Promise<void> {
  }
 }

-// --- Start Server and Background Tasks ---
+// Start Server and Background Tasks
 async function startServer(): Promise<void> {
  const redis = getRedisClient();
  if (!redis) {
--- a/models/activity.ts
+++ b/models/activity.ts
@@ -1,10 +1,10 @@
 // src/models/activity.ts
 export interface ActivityData {
  // Include all common properties
-  id?: string | null | undefined;
+  id?: string | null;
  name?: string | null;
  description?: string | null;
-  photo?: string | null | undefined;
+  photo?: string | null;
  academicYear?: string | null;
  category?: string | null;
  isPreSignup?: boolean | null;
--- a/package.json
+++ b/package.json
@@ -14,6 +14,7 @@
  "dependencies": {
    "axios": "^1.9.0",
    "cors": "^2.8.5",
+    "crypto": "^1.0.1",
    "dotenv": "^16.5.0",
    "express": "^5.1.0",
    "p-limit": "^6.2.0",
--- a/services/cache-manager.ts
+++ b/services/cache-manager.ts
@@ -77,7 +77,6 @@ async function processAndCacheActivity(activityId: string): Promise<ActivityData
        }
      }
    }
-
    structuredActivity.lastCheck = new Date().toISOString();
    await setActivityData(activityId, structuredActivity);
    return structuredActivity;
--- a/services/s3-service.ts
+++ b/services/s3-service.ts
@@ -3,6 +3,7 @@ import { S3Client } from "bun";
 import { v4 as uuidv4 } from 'uuid';
 import { config } from 'dotenv';
 import sharp from 'sharp';
+import crypto from 'crypto';
 import { logger } from '../utils/logger';
 import { decodeBase64Image } from '../utils/image-processor';

@@ -38,9 +39,10 @@ if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {

 /**
 * Uploads an image from a base64 string to S3, converting it to AVIF format.
+ * Uses MD5 checksum as filename and checks for duplicates before uploading.
 * @param base64Data - The base64 content (without the data URI prefix)
 * @param originalFormat - The image format (e.g., 'png', 'jpeg')
- * @param activityId - The activity ID, used for naming
+ * @param activityId - The activity ID, used for logging purposes
 * @returns The public URL of the uploaded image or null on error
 */
 export async function uploadImageFromBase64(
@@ -56,11 +58,9 @@ export async function uploadImageFromBase64(
    logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
    return null;
  }
-
  try {
    // First decode the base64 image
    const imageBuffer = decodeBase64Image(base64Data);
-        
    // Convert to AVIF format with quality 80 using Sharp
    const avifBuffer = await sharp(imageBuffer)
      .avif({ 
@@ -70,20 +70,26 @@ export async function uploadImageFromBase64(
        // effort: 4,
      })
      .toBuffer();
-        
-        // Use .avif extension for the object key
-        const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`;
-        
-        // Using Bun's S3Client file API
+    // Calculate MD5 checksum of the converted AVIF image
+    const md5Hash = crypto.createHash('md5').update(avifBuffer).digest('hex');
+    const objectKey = `${PUBLIC_URL_FILE_PREFIX}/${md5Hash}.avif`;
+    // Check if file with this checksum already exists
    const s3File = s3Client.file(objectKey);
+    const exists = await s3File.exists();
    
+    if (exists) {
+      const publicUrl = constructS3Url(objectKey);
+      logger.info(`Image already exists in S3 (MD5: ${md5Hash}), returning existing URL: ${publicUrl}`);
+      return publicUrl;
+    }
+    // File doesn't exist, proceed with upload
    await s3File.write(avifBuffer, {
      type: 'image/avif',
      acl: 'public-read'
    });
    
    const publicUrl = constructS3Url(objectKey);
-        logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`);
+    logger.info(`Image uploaded to S3 as AVIF (MD5: ${md5Hash}): ${publicUrl}`);
    return publicUrl;
  } catch (error) {
    logger.error(`S3 Upload Error for activity ${activityId}:`, error);
@@ -116,7 +122,6 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
        startAfter,
        maxKeys: 1000
      });
-            
      if (result.contents) {
        // Add keys to our array, filtering out "directories"
        result.contents.forEach(item => {
@@ -124,21 +129,17 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
            objectKeys.push(item.key);
          }
        });
-                
        // Get the last key for pagination
        if (result.contents?.length > 0) {
          startAfter = result.contents[result.contents.length - 1]?.key;
        }
      }
-            
      isTruncated = result.isTruncated || false;
-            
      // Safety check to prevent infinite loops
      if (result.contents?.length === 0) {
        break;
      }
    }
-        
    logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
    return objectKeys;
  } catch (error) {
@@ -161,7 +162,6 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
    logger.info('No objects to delete from S3.');
    return true;
  }
-
  try {
    // With Bun's S3Client, we need to delete objects one by one
    // Process in batches of 100 for better performance
@@ -171,12 +171,10 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
    
    for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) {
      const batch = objectKeysArray.slice(i, i + BATCH_SIZE);
-            
      // Process batch in parallel
      const results = await Promise.allSettled(
        batch.map(key => s3Client!.delete(key))
      );
-            
      // Count successes and failures
      for (const result of results) {
        if (result.status === 'fulfilled') {
@@ -187,7 +185,6 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
        }
      }
    }
-        
    logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
    return errorCount === 0; // True if all succeeded
  } catch (error) {
@@ -205,7 +202,6 @@ export function constructS3Url(objectKey: string): string {
  if (!S3_ENDPOINT || !BUCKET_NAME) {
    return '';
  }
-    
  // Ensure S3_ENDPOINT does not end with a slash
  const s3Base = S3_ENDPOINT.replace(/\/$/, '');
  // Ensure BUCKET_NAME does not start or end with a slash
Author	SHA1	Message	Date
JamesFlare1212	7ba5f8f00f	feat: skip duplicate image on remote	2025-05-12 23:46:25 -04:00
JamesFlare1212	8598571f72	improve: code structure	2025-05-12 21:45:57 -04:00