JamesFlare/dsas-cca-backend-bun
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(s3): remove automatic image deletion, users manage S3 files

Browse Source
This commit is contained in:
JamesFlare1212
2026-04-08 10:29:27 -04:00
parent 1e234624fb
commit 78c050a6fa
2 changed files with 1 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
services/cache-manager.ts
View File

@@ -11,7 +11,7 @@ import {
getAllActivityKeys, getAllActivityKeys,
ACTIVITY_KEY_PREFIX ACTIVITY_KEY_PREFIX
} from './redis-service'; } from './redis-service';
import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service'; import { uploadImageFromBase64, listS3Objects, constructS3Url } from './s3-service';
import { extractBase64Image } from '../utils/image-processor'; import { extractBase64Image } from '../utils/image-processor';
import { logger } from '../utils/logger'; import { logger } from '../utils/logger';
import { BatchProcessor, executeWithConcurrencyAndProgress } from '../utils/semaphore'; import { BatchProcessor, executeWithConcurrencyAndProgress } from '../utils/semaphore';
@@ -231,7 +231,6 @@ export async function updateStaleClubs(): Promise<void> {
if (staleActivityIds.length === 0) { if (staleActivityIds.length === 0) {
logger.info('No stale activities found. Skipping update.'); logger.info('No stale activities found. Skipping update.');
await cleanupOrphanedS3Images();
logger.info('Stale club check finished.'); logger.info('Stale club check finished.');
return; return;
} }
@@ -261,8 +260,6 @@ export async function updateStaleClubs(): Promise<void> {
// Process stale activities concurrently // Process stale activities concurrently
await processor.process(staleActivityIds); await processor.process(staleActivityIds);
await cleanupOrphanedS3Images();
logger.info('Stale club check finished.'); logger.info('Stale club check finished.');
} }
@@ -311,128 +308,3 @@ export async function initializeOrUpdateStaffCache(forceUpdate: boolean = false)
logger.error('Error initializing or updating staff cache:', error); logger.error('Error initializing or updating staff cache:', error);
} }
} }
/**
* Clean up orphaned S3 images
*
* FIX: Changed from URL-based matching to object key-based matching.
* Previous bug: URL mismatch between S3_PUBLIC_URL and S3_ENDPOINT caused
* valid images to be incorrectly marked as orphaned and deleted.
*
* New approach: Extract object keys from Redis-stored URLs and compare
* directly with S3 object keys. This is immune to URL configuration changes.
*/
export async function cleanupOrphanedS3Images(): Promise<void> {
logger.info('Starting S3 orphan image cleanup...');
const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : '';
try {
const referencedObjectKeys = new Set<string>();
const allActivityRedisKeys = await getAllActivityKeys();
let photoCount = 0;
let httpUrlCount = 0;
let base64Count = 0;
let nullCount = 0;
let extractedKeyCount = 0;
let failedExtractionCount = 0;
let sampleHttpUrl = null;
let sampleBase64Url = null;
let sampleExtractedKey = null;
for (const redisKey of allActivityRedisKeys) {
const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length);
const activityData = await getActivityData(activityId);
if (activityData) {
if (activityData.photo) {
photoCount++;
if (typeof activityData.photo === 'string') {
if (activityData.photo.startsWith('http')) {
httpUrlCount++;
const objectKey = extractObjectKeyFromUrl(activityData.photo);
if (objectKey) {
referencedObjectKeys.add(objectKey);
extractedKeyCount++;
if (!sampleExtractedKey) sampleExtractedKey = { url: activityData.photo, key: objectKey };
} else {
failedExtractionCount++;
}
if (!sampleHttpUrl) sampleHttpUrl = activityData.photo;
} else if (activityData.photo.startsWith('data:image')) {
base64Count++;
if (!sampleBase64Url) sampleBase64Url = activityData.photo.substring(0, 50) + '...';
}
}
} else {
nullCount++;
}
}
}
logger.info(`Found ${referencedObjectKeys.size} unique S3 object keys referenced in Redis.`);
logger.info(`Photo stats: ${httpUrlCount} HTTP URLs, ${base64Count} base64, ${nullCount} null/empty out of ${photoCount} activities with photo field.`);
logger.info(`Key extraction: ${extractedKeyCount} succeeded, ${failedExtractionCount} failed.`);
if (sampleHttpUrl) logger.debug(`Sample HTTP URL: ${sampleHttpUrl}`);
if (sampleExtractedKey) logger.debug(`Sample extracted key: URL="${sampleExtractedKey.url}" → key="${sampleExtractedKey.key}"`);
if (sampleBase64Url) logger.debug(`Sample base64 URL: ${sampleBase64Url}`);
if (httpUrlCount === 0 && base64Count > 0) {
logger.error('CRITICAL: All photos are base64 format! S3 upload may be failing or photo field not updated.');
}
if (failedExtractionCount > 0) {
logger.warn(`Failed to extract ${failedExtractionCount} object keys from URLs. These images won't be protected from deletion.`);
}
const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix);
if (!s3ObjectKeys || s3ObjectKeys.length === 0) {
logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`);
return;
}
logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`);
const orphanedObjectKeys: string[] = [];
let matchedCount = 0;
let notFoundCount = 0;
let firstOrphanExample = null;
for (const objectKey of s3ObjectKeys) {
if (referencedObjectKeys.has(objectKey)) {
matchedCount++;
} else {
notFoundCount++;
if (!firstOrphanExample) {
firstOrphanExample = objectKey;
}
}
}
logger.info(`S3 object key matching: ${matchedCount} matched, ${notFoundCount} not found (orphaned).`);
if (notFoundCount > 0 && firstOrphanExample) {
logger.debug(`Example orphaned object: key="${firstOrphanExample}"`);
}
if (matchedCount === 0 && s3ObjectKeys.length > 0 && referencedObjectKeys.size > 0) {
logger.error('CRITICAL: No matches found between S3 objects and Redis references!');
logger.error('CRITICAL: Sample from Redis keys: ' + [...referencedObjectKeys].slice(0, 3).join(', '));
logger.error('CRITICAL: Sample from S3 keys: ' + s3ObjectKeys.slice(0, 3).join(', '));
logger.error('CRITICAL: This indicates a serious bug in object key extraction or S3 listing.');
}
for (const objectKey of s3ObjectKeys) {
if (!referencedObjectKeys.has(objectKey)) {
orphanedObjectKeys.push(objectKey);
}
}
if (orphanedObjectKeys.length > 0) {
logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`);
await deleteS3Objects(orphanedObjectKeys);
} else {
logger.info('No orphaned S3 images found after comparison.');
}
logger.info('S3 orphan image cleanup finished.');
} catch (error) {
logger.error('Error during S3 orphan image cleanup:', error);
}
}

35
services/s3-service.ts
View File

@@ -149,41 +149,6 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
} }
} }
/**
* Deletes multiple objects from S3.
* @param objectKeysArray - Array of object keys to delete
* @returns True if successful or partially successful, false on major error
*/
export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolean> {
if (!s3Client) {
logger.warn('S3 client not configured. Cannot delete objects.');
return false;
}
if (!objectKeysArray || objectKeysArray.length === 0) {
logger.info('No objects to delete from S3.');
return true;
}
try {
let successCount = 0;
let errorCount = 0;
for (const key of objectKeysArray) {
try {
await s3Client.delete(key);
successCount++;
} catch (error) {
errorCount++;
logger.error(`Failed to delete object ${key}:`, error);
}
}
logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
return errorCount === 0; // True if all succeeded
} catch (error) {
logger.error('S3 DeleteObjects Error:', error);
return false;
}
}
/** /**
* Constructs the public S3 URL for an object key. * Constructs the public S3 URL for an object key.
* Uses S3_PUBLIC_URL if set (reverse proxy scenario), otherwise uses S3_ENDPOINT. * Uses S3_PUBLIC_URL if set (reverse proxy scenario), otherwise uses S3_ENDPOINT.