feat: redis cache and detach image into s3
This commit is contained in:
JamesFlare1212
186
services/s3-service.mjs
Normal file
186
services/s3-service.mjs
Normal file
@@ -0,0 +1,186 @@
|
||||
// services/s3-service.mjs
|
||||
import { S3Client, PutObjectCommand, ListObjectsV2Command, DeleteObjectsCommand } from '@aws-sdk/client-s3';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import dotenv from 'dotenv';
|
||||
import { logger } from '../utils/logger.mjs';
|
||||
import { decodeBase64Image } from '../utils/image-processor.mjs';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const S3_ENDPOINT = process.env.S3_ENDPOINT;
|
||||
const S3_REGION = process.env.S3_REGION;
|
||||
const S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID;
|
||||
const S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY;
|
||||
const BUCKET_NAME = process.env.S3_BUCKET_NAME;
|
||||
const PUBLIC_URL_FILE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').replace(/\/$/, ''); // Ensures no trailing slash
|
||||
|
||||
let s3Client;
|
||||
|
||||
if (S3_ENDPOINT && S3_REGION && S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {
|
||||
s3Client = new S3Client({
|
||||
endpoint: S3_ENDPOINT,
|
||||
region: S3_REGION,
|
||||
credentials: {
|
||||
accessKeyId: S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: S3_SECRET_ACCESS_KEY,
|
||||
},
|
||||
forcePathStyle: true, // Important for MinIO and some S3-compatibles
|
||||
});
|
||||
} else {
|
||||
logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.');
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Uploads an image from a base64 string to S3.
|
||||
* @param {string} base64Data The base64 content (without the data URI prefix).
|
||||
* @param {string} originalFormat The image format (e.g., 'png', 'jpeg').
|
||||
* @param {string} activityId The activity ID, used for naming.
|
||||
* @returns {Promise<string|null>} The public URL of the uploaded image or null on error.
|
||||
*/
|
||||
export async function uploadImageFromBase64(base64Data, originalFormat, activityId) {
|
||||
if (!s3Client) {
|
||||
logger.warn('S3 client not configured. Cannot upload image.');
|
||||
return null;
|
||||
}
|
||||
if (!base64Data || !originalFormat || !activityId) {
|
||||
logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const imageBuffer = decodeBase64Image(base64Data);
|
||||
// Ensure PUBLIC_URL_FILE_PREFIX is part of the key
|
||||
const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.${originalFormat}`;
|
||||
|
||||
const params = {
|
||||
Bucket: BUCKET_NAME,
|
||||
Key: objectKey,
|
||||
Body: imageBuffer,
|
||||
ContentType: `image/${originalFormat}`,
|
||||
ACL: 'public-read',
|
||||
};
|
||||
|
||||
await s3Client.send(new PutObjectCommand(params));
|
||||
const publicUrl = constructS3Url(objectKey);
|
||||
logger.info(`Image uploaded to S3: ${publicUrl}`);
|
||||
return publicUrl;
|
||||
} catch (error) {
|
||||
logger.error(`S3 Upload Error for activity ${activityId}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all objects in the S3 bucket under a specific prefix.
|
||||
* @param {string} prefix The prefix to filter objects by (e.g., S3_PUBLIC_URL_PREFIX + '/').
|
||||
* @returns {Promise<Array<string>>} A list of object keys.
|
||||
*/
|
||||
export async function listS3Objects(prefix) {
|
||||
if (!s3Client) {
|
||||
logger.warn('S3 client not configured. Cannot list objects.');
|
||||
return [];
|
||||
}
|
||||
const objectKeys = [];
|
||||
let isTruncated = true;
|
||||
let continuationToken;
|
||||
|
||||
logger.debug(`Listing objects from S3 with prefix: "${prefix}"`);
|
||||
const listCommandInput = { // Renamed to avoid conflict if command is redefined in loop
|
||||
Bucket: BUCKET_NAME,
|
||||
Prefix: prefix,
|
||||
};
|
||||
|
||||
try {
|
||||
while (isTruncated) {
|
||||
if (continuationToken) {
|
||||
listCommandInput.ContinuationToken = continuationToken;
|
||||
}
|
||||
const command = new ListObjectsV2Command(listCommandInput);
|
||||
const { Contents, IsTruncated: NextIsTruncated, NextContinuationToken } = await s3Client.send(command);
|
||||
|
||||
if (Contents) {
|
||||
Contents.forEach(item => {
|
||||
if (item.Key && !item.Key.endsWith('/')) { // Ensure it's a file, not a pseudo-directory
|
||||
objectKeys.push(item.Key);
|
||||
}
|
||||
});
|
||||
}
|
||||
isTruncated = NextIsTruncated;
|
||||
continuationToken = NextContinuationToken;
|
||||
}
|
||||
logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
|
||||
return objectKeys;
|
||||
} catch (error) {
|
||||
logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Deletes multiple objects from S3.
|
||||
* @param {Array<string>} objectKeysArray Array of object keys to delete.
|
||||
* @returns {Promise<boolean>} True if successful or partially successful, false on major error.
|
||||
*/
|
||||
export async function deleteS3Objects(objectKeysArray) {
|
||||
if (!s3Client) {
|
||||
logger.warn('S3 client not configured. Cannot delete objects.');
|
||||
return false;
|
||||
}
|
||||
if (!objectKeysArray || objectKeysArray.length === 0) {
|
||||
logger.info('No objects to delete from S3.');
|
||||
return true;
|
||||
}
|
||||
|
||||
const MAX_DELETE_COUNT = 1000; // S3 API limit
|
||||
let allDeletionsSuccessful = true;
|
||||
|
||||
for (let i = 0; i < objectKeysArray.length; i += MAX_DELETE_COUNT) {
|
||||
const chunk = objectKeysArray.slice(i, i + MAX_DELETE_COUNT);
|
||||
const deleteParams = {
|
||||
Bucket: BUCKET_NAME,
|
||||
Delete: {
|
||||
Objects: chunk.map(key => ({ Key: key })),
|
||||
Quiet: false, // We want error details
|
||||
},
|
||||
};
|
||||
try {
|
||||
const command = new DeleteObjectsCommand(deleteParams);
|
||||
const output = await s3Client.send(command);
|
||||
if (output.Errors && output.Errors.length > 0) {
|
||||
allDeletionsSuccessful = false;
|
||||
output.Errors.forEach(err => {
|
||||
logger.error(`S3 Delete Error for key ${err.Key}: ${err.Message}`);
|
||||
});
|
||||
}
|
||||
if (output.Deleted && output.Deleted.length > 0) {
|
||||
logger.info(`Successfully submitted deletion for ${output.Deleted.length} objects from S3 chunk (some might have failed, check individual errors).`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('S3 DeleteObjects Command Error for a chunk:', error);
|
||||
allDeletionsSuccessful = false;
|
||||
}
|
||||
}
|
||||
if (allDeletionsSuccessful && objectKeysArray.length > 0) {
|
||||
logger.info(`Finished S3 deletion request for ${objectKeysArray.length} keys.`);
|
||||
} else if (objectKeysArray.length > 0) {
|
||||
logger.warn(`S3 deletion request for ${objectKeysArray.length} keys completed with some errors.`);
|
||||
}
|
||||
return allDeletionsSuccessful;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs the public S3 URL for an object key.
|
||||
* @param {string} objectKey The key of the object in S3.
|
||||
* @returns {string} The full public URL.
|
||||
*/
|
||||
export function constructS3Url(objectKey) {
|
||||
// Ensure S3_ENDPOINT does not end with a slash
|
||||
const s3Base = S3_ENDPOINT.replace(/\/$/, '');
|
||||
// Ensure BUCKET_NAME does not start or end with a slash
|
||||
const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, '');
|
||||
// Ensure objectKey does not start with a slash
|
||||
const key = objectKey.replace(/^\//, '');
|
||||
return `${s3Base}/${bucket}/${key}`;
|
||||
}
|
||||
Reference in New Issue
Block a user