diff --git a/services/cache-manager.ts b/services/cache-manager.ts index b35bbd1..27a7728 100644 --- a/services/cache-manager.ts +++ b/services/cache-manager.ts @@ -11,7 +11,7 @@ import { getAllActivityKeys, ACTIVITY_KEY_PREFIX } from './redis-service'; -import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service'; +import { uploadImageFromBase64, listS3Objects, constructS3Url } from './s3-service'; import { extractBase64Image } from '../utils/image-processor'; import { logger } from '../utils/logger'; import { BatchProcessor, executeWithConcurrencyAndProgress } from '../utils/semaphore'; @@ -231,7 +231,6 @@ export async function updateStaleClubs(): Promise { if (staleActivityIds.length === 0) { logger.info('No stale activities found. Skipping update.'); - await cleanupOrphanedS3Images(); logger.info('Stale club check finished.'); return; } @@ -261,8 +260,6 @@ export async function updateStaleClubs(): Promise { // Process stale activities concurrently await processor.process(staleActivityIds); - await cleanupOrphanedS3Images(); - logger.info('Stale club check finished.'); } @@ -311,128 +308,3 @@ export async function initializeOrUpdateStaffCache(forceUpdate: boolean = false) logger.error('Error initializing or updating staff cache:', error); } } - -/** - * Clean up orphaned S3 images - * - * FIX: Changed from URL-based matching to object key-based matching. - * Previous bug: URL mismatch between S3_PUBLIC_URL and S3_ENDPOINT caused - * valid images to be incorrectly marked as orphaned and deleted. - * - * New approach: Extract object keys from Redis-stored URLs and compare - * directly with S3 object keys. This is immune to URL configuration changes. - */ -export async function cleanupOrphanedS3Images(): Promise { - logger.info('Starting S3 orphan image cleanup...'); - const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : ''; - - try { - const referencedObjectKeys = new Set(); - const allActivityRedisKeys = await getAllActivityKeys(); - - let photoCount = 0; - let httpUrlCount = 0; - let base64Count = 0; - let nullCount = 0; - let extractedKeyCount = 0; - let failedExtractionCount = 0; - let sampleHttpUrl = null; - let sampleBase64Url = null; - let sampleExtractedKey = null; - - for (const redisKey of allActivityRedisKeys) { - const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length); - const activityData = await getActivityData(activityId); - - if (activityData) { - if (activityData.photo) { - photoCount++; - if (typeof activityData.photo === 'string') { - if (activityData.photo.startsWith('http')) { - httpUrlCount++; - const objectKey = extractObjectKeyFromUrl(activityData.photo); - if (objectKey) { - referencedObjectKeys.add(objectKey); - extractedKeyCount++; - if (!sampleExtractedKey) sampleExtractedKey = { url: activityData.photo, key: objectKey }; - } else { - failedExtractionCount++; - } - if (!sampleHttpUrl) sampleHttpUrl = activityData.photo; - } else if (activityData.photo.startsWith('data:image')) { - base64Count++; - if (!sampleBase64Url) sampleBase64Url = activityData.photo.substring(0, 50) + '...'; - } - } - } else { - nullCount++; - } - } - } - - logger.info(`Found ${referencedObjectKeys.size} unique S3 object keys referenced in Redis.`); - logger.info(`Photo stats: ${httpUrlCount} HTTP URLs, ${base64Count} base64, ${nullCount} null/empty out of ${photoCount} activities with photo field.`); - logger.info(`Key extraction: ${extractedKeyCount} succeeded, ${failedExtractionCount} failed.`); - if (sampleHttpUrl) logger.debug(`Sample HTTP URL: ${sampleHttpUrl}`); - if (sampleExtractedKey) logger.debug(`Sample extracted key: URL="${sampleExtractedKey.url}" → key="${sampleExtractedKey.key}"`); - if (sampleBase64Url) logger.debug(`Sample base64 URL: ${sampleBase64Url}`); - if (httpUrlCount === 0 && base64Count > 0) { - logger.error('CRITICAL: All photos are base64 format! S3 upload may be failing or photo field not updated.'); - } - if (failedExtractionCount > 0) { - logger.warn(`Failed to extract ${failedExtractionCount} object keys from URLs. These images won't be protected from deletion.`); - } - - const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix); - if (!s3ObjectKeys || s3ObjectKeys.length === 0) { - logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`); - return; - } - - logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`); - - const orphanedObjectKeys: string[] = []; - let matchedCount = 0; - let notFoundCount = 0; - let firstOrphanExample = null; - - for (const objectKey of s3ObjectKeys) { - if (referencedObjectKeys.has(objectKey)) { - matchedCount++; - } else { - notFoundCount++; - if (!firstOrphanExample) { - firstOrphanExample = objectKey; - } - } - } - - logger.info(`S3 object key matching: ${matchedCount} matched, ${notFoundCount} not found (orphaned).`); - if (notFoundCount > 0 && firstOrphanExample) { - logger.debug(`Example orphaned object: key="${firstOrphanExample}"`); - } - if (matchedCount === 0 && s3ObjectKeys.length > 0 && referencedObjectKeys.size > 0) { - logger.error('CRITICAL: No matches found between S3 objects and Redis references!'); - logger.error('CRITICAL: Sample from Redis keys: ' + [...referencedObjectKeys].slice(0, 3).join(', ')); - logger.error('CRITICAL: Sample from S3 keys: ' + s3ObjectKeys.slice(0, 3).join(', ')); - logger.error('CRITICAL: This indicates a serious bug in object key extraction or S3 listing.'); - } - - for (const objectKey of s3ObjectKeys) { - if (!referencedObjectKeys.has(objectKey)) { - orphanedObjectKeys.push(objectKey); - } - } - - if (orphanedObjectKeys.length > 0) { - logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`); - await deleteS3Objects(orphanedObjectKeys); - } else { - logger.info('No orphaned S3 images found after comparison.'); - } - - logger.info('S3 orphan image cleanup finished.'); - } catch (error) { - logger.error('Error during S3 orphan image cleanup:', error); - } -} diff --git a/services/s3-service.ts b/services/s3-service.ts index 24689e5..17dfb1f 100644 --- a/services/s3-service.ts +++ b/services/s3-service.ts @@ -149,41 +149,6 @@ export async function listS3Objects(prefix: string): Promise { } } -/** - * Deletes multiple objects from S3. - * @param objectKeysArray - Array of object keys to delete - * @returns True if successful or partially successful, false on major error - */ -export async function deleteS3Objects(objectKeysArray: string[]): Promise { - if (!s3Client) { - logger.warn('S3 client not configured. Cannot delete objects.'); - return false; - } - if (!objectKeysArray || objectKeysArray.length === 0) { - logger.info('No objects to delete from S3.'); - return true; - } - try { - let successCount = 0; - let errorCount = 0; - - for (const key of objectKeysArray) { - try { - await s3Client.delete(key); - successCount++; - } catch (error) { - errorCount++; - logger.error(`Failed to delete object ${key}:`, error); - } - } - logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`); - return errorCount === 0; // True if all succeeded - } catch (error) { - logger.error('S3 DeleteObjects Error:', error); - return false; - } -} - /** * Constructs the public S3 URL for an object key. * Uses S3_PUBLIC_URL if set (reverse proxy scenario), otherwise uses S3_ENDPOINT.