diff --git a/bun.lock b/bun.lock index 1c1f8a4..ff3f6e1 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,7 @@ "dependencies": { "axios": "^1.9.0", "cors": "^2.8.5", + "crypto": "^1.0.1", "dotenv": "^16.5.0", "express": "^5.1.0", "p-limit": "^6.2.0", @@ -104,6 +105,8 @@ "cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="], + "crypto": ["crypto@1.0.1", "", {}, "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="], + "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="], "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], diff --git a/engage-api/struct-activity.ts b/engage-api/struct-activity.ts index 4d6fb28..c35b12b 100644 --- a/engage-api/struct-activity.ts +++ b/engage-api/struct-activity.ts @@ -254,4 +254,4 @@ export async function structActivityData(rawActivityData: RawActivityData): Prom } await postProcess(structuredActivityData); return structuredActivityData; -} +} \ No newline at end of file diff --git a/index.ts b/index.ts index bc05eab..b25f58b 100644 --- a/index.ts +++ b/index.ts @@ -412,7 +412,7 @@ async function performBackgroundTasks(): Promise { } } -// --- Start Server and Background Tasks --- +// Start Server and Background Tasks async function startServer(): Promise { const redis = getRedisClient(); if (!redis) { diff --git a/package.json b/package.json index 5637a2d..80197f0 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "dependencies": { "axios": "^1.9.0", "cors": "^2.8.5", + "crypto": "^1.0.1", "dotenv": "^16.5.0", "express": "^5.1.0", "p-limit": "^6.2.0", diff --git a/services/s3-service.ts b/services/s3-service.ts index 37c0abb..c620d75 100644 --- a/services/s3-service.ts +++ b/services/s3-service.ts @@ -3,6 +3,7 @@ import { S3Client } from "bun"; import { v4 as uuidv4 } from 'uuid'; import { config } from 'dotenv'; import sharp from 'sharp'; +import crypto from 'crypto'; import { logger } from '../utils/logger'; import { decodeBase64Image } from '../utils/image-processor'; @@ -38,9 +39,10 @@ if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) { /** * Uploads an image from a base64 string to S3, converting it to AVIF format. + * Uses MD5 checksum as filename and checks for duplicates before uploading. * @param base64Data - The base64 content (without the data URI prefix) * @param originalFormat - The image format (e.g., 'png', 'jpeg') - * @param activityId - The activity ID, used for naming + * @param activityId - The activity ID, used for logging purposes * @returns The public URL of the uploaded image or null on error */ export async function uploadImageFromBase64( @@ -56,7 +58,6 @@ export async function uploadImageFromBase64( logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId'); return null; } - try { // First decode the base64 image const imageBuffer = decodeBase64Image(base64Data); @@ -69,18 +70,26 @@ export async function uploadImageFromBase64( // effort: 4, }) .toBuffer(); - // Use .avif extension for the object key - const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`; - // Using Bun's S3Client file API + // Calculate MD5 checksum of the converted AVIF image + const md5Hash = crypto.createHash('md5').update(avifBuffer).digest('hex'); + const objectKey = `${PUBLIC_URL_FILE_PREFIX}/${md5Hash}.avif`; + // Check if file with this checksum already exists const s3File = s3Client.file(objectKey); + const exists = await s3File.exists(); + if (exists) { + const publicUrl = constructS3Url(objectKey); + logger.info(`Image already exists in S3 (MD5: ${md5Hash}), returning existing URL: ${publicUrl}`); + return publicUrl; + } + // File doesn't exist, proceed with upload await s3File.write(avifBuffer, { type: 'image/avif', acl: 'public-read' }); const publicUrl = constructS3Url(objectKey); - logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`); + logger.info(`Image uploaded to S3 as AVIF (MD5: ${md5Hash}): ${publicUrl}`); return publicUrl; } catch (error) { logger.error(`S3 Upload Error for activity ${activityId}:`, error); @@ -113,7 +122,6 @@ export async function listS3Objects(prefix: string): Promise { startAfter, maxKeys: 1000 }); - if (result.contents) { // Add keys to our array, filtering out "directories" result.contents.forEach(item => { @@ -126,14 +134,12 @@ export async function listS3Objects(prefix: string): Promise { startAfter = result.contents[result.contents.length - 1]?.key; } } - isTruncated = result.isTruncated || false; // Safety check to prevent infinite loops if (result.contents?.length === 0) { break; } } - logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`); return objectKeys; } catch (error) { @@ -156,7 +162,6 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise