JamesFlare/dsas-cca-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: redis cache and detach image into s3

Browse Source
This commit is contained in:
JamesFlare1212
2025-05-09 19:43:01 -04:00
parent 99d1ee0a1e
commit f7252345f3
14 changed files with 2302 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,3 +1,4 @@
node_modules
nkcs-engage.cookie.txt
.env
redis_data

25
docker-compose.yaml
View File

@@ -1,5 +1,5 @@
services:
dsas-cca-backend:
app:
build:
context: .
dockerfile: Dockerfile
@@ -9,3 +9,26 @@ services:
env_file:
- .env
restart: unless-stopped
depends_on:
- redis
networks:
- cca_network
redis:
image: "redis:7.2-alpine"
container_name: dsas-cca-redis
command: redis-server --requirepass "dsas-cca"
volumes:
- ./redis_data:/data
restart: unless-stopped
networks:
- cca_network
healthcheck:
test: ["CMD", "redis-cli", "-a", "dsas-cca", "ping"]
interval: 10s
timeout: 5s
retries: 5
networks:
cca_network:
driver: bridge

89
engage-api/get-activity.mjs
View File

@@ -1,9 +1,9 @@
// get-activity.mjs
import axios from 'axios';
import fs from 'fs/promises'; // Using fs.promises directly
import path from 'path';
import { fileURLToPath } from 'url';
import { logger } from '../utils/logger.mjs';
// --- Replicating __dirname for ESM ---
const __filename = fileURLToPath(import.meta.url);
@@ -25,21 +25,21 @@ class AuthenticationError extends Error {
// --- Cookie Cache Helper Functions ---
async function loadCachedCookie() {
if (_inMemoryCookie) {
console.log("Using in-memory cached cookie.");
logger.debug("Using in-memory cached cookie.");
return _inMemoryCookie;
}
try {
const cookieFromFile = await fs.readFile(COOKIE_FILE_PATH, 'utf8');
if (cookieFromFile) {
_inMemoryCookie = cookieFromFile;
console.log("Loaded cookie from file cache.");
logger.debug("Loaded cookie from file cache.");
return _inMemoryCookie;
}
} catch (err) {
if (err.code === 'ENOENT') {
console.log("Cookie cache file not found. No cached cookie loaded.");
logger.debug("Cookie cache file not found. No cached cookie loaded.");
} else {
console.warn("Error loading cookie from file:", err.message);
logger.warn("Error loading cookie from file:", err.message);
}
}
return null;
@@ -47,15 +47,15 @@ async function loadCachedCookie() {
async function saveCookieToCache(cookieString) {
if (!cookieString) {
console.warn("Attempted to save an empty or null cookie. Aborting save.");
logger.warn("Attempted to save an empty or null cookie. Aborting save.");
return;
}
_inMemoryCookie = cookieString;
try {
await fs.writeFile(COOKIE_FILE_PATH, cookieString, 'utf8');
console.log("Cookie saved to file cache.");
logger.debug("Cookie saved to file cache.");
} catch (err) {
console.error("Error saving cookie to file:", err.message);
logger.error("Error saving cookie to file:", err.message);
}
}
@@ -63,19 +63,19 @@ async function clearCookieCache() {
_inMemoryCookie = null;
try {
await fs.unlink(COOKIE_FILE_PATH);
console.log("Cookie cache file deleted.");
logger.debug("Cookie cache file deleted.");
} catch (err) {
if (err.code !== 'ENOENT') {
console.error("Error deleting cookie file:", err.message);
logger.error("Error deleting cookie file:", err.message);
} else {
console.log("Cookie cache file did not exist, no need to delete.");
logger.debug("Cookie cache file did not exist, no need to delete.");
}
}
}
async function testCookieValidity(cookieString) {
if (!cookieString) return false;
console.log("Testing cookie validity...");
logger.debug("Testing cookie validity...");
try {
const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
const headers = {
@@ -85,14 +85,14 @@ async function testCookieValidity(cookieString) {
};
const payload = { "activityID": "3350" };
await axios.post(url, payload, { headers, timeout: 10000 });
console.log("Cookie test successful (API responded 2xx). Cookie is valid.");
logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
return true;
} catch (error) {
console.warn("Cookie validity test failed.");
logger.warn("Cookie validity test failed.");
if (error.response) {
console.warn(`Cookie test API response status: ${error.response.status}. Cookie is likely invalid or expired.`);
logger.warn(`Cookie test API response status: ${error.response.status}. Cookie is likely invalid or expired.`);
} else {
console.warn(`Cookie test failed due to network or other error: ${error.message}`);
logger.warn(`Cookie test failed due to network or other error: ${error.message}`);
}
return false;
}
@@ -109,14 +109,14 @@ async function getSessionId() {
if (setCookieHeader && setCookieHeader.length > 0) {
const sessionIdCookie = setCookieHeader.find(cookie => cookie.trim().startsWith('ASP.NET_SessionId='));
if (sessionIdCookie) {
console.log('Debugging - ASP.NET_SessionId created');
logger.debug('ASP.NET_SessionId created');
return sessionIdCookie.split(';')[0];
}
}
console.error("No ASP.NET_SessionId cookie found in Set-Cookie header.");
logger.error("No ASP.NET_SessionId cookie found in Set-Cookie header.");
return null;
} catch (error) {
console.error(`Error in getSessionId: ${error.response ? `${error.response.status} - ${error.response.statusText}` : error.message}`);
logger.error(`Error in getSessionId: ${error.response ? `${error.response.status} - ${error.response.statusText}` : error.message}`);
throw error;
}
}
@@ -133,7 +133,7 @@ async function getMSAUTH(sessionId, userName, userPwd, templateFilePath) {
'User-Agent': 'Mozilla/5.0 (Node.js DSAS-CCA get-activity Module)',
'Referer': 'https://engage.nkcswx.cn/Login.aspx'
};
console.log('Debugging - Getting .ASPXFORMSAUTH');
logger.debug('Getting .ASPXFORMSAUTH');
const response = await axios.post(url, postData, {
headers, maxRedirects: 0,
validateStatus: (status) => status >= 200 && status < 400
@@ -153,21 +153,21 @@ async function getMSAUTH(sessionId, userName, userPwd, templateFilePath) {
}
}
if (formsAuthCookieValue) {
console.log('Debugging - .ASPXFORMSAUTH cookie obtained.');
logger.debug('.ASPXFORMSAUTH cookie obtained.');
return formsAuthCookieValue;
} else {
console.error("No valid .ASPXFORMSAUTH cookie found. Headers:", setCookieHeader || "none");
logger.error("No valid .ASPXFORMSAUTH cookie found. Headers:", setCookieHeader || "none");
return null;
}
} catch (error) {
if (error.code === 'ENOENT') console.error(`Error: Template file '${templateFilePath}' not found.`);
else console.error(`Error in getMSAUTH: ${error.message}`);
if (error.code === 'ENOENT') logger.error(`Error: Template file '${templateFilePath}' not found.`);
else logger.error(`Error in getMSAUTH: ${error.message}`);
throw error;
}
}
async function getCompleteCookies(userName, userPwd, templateFilePath) {
console.log('Debugging - Attempting to get complete cookie string (login process).');
logger.debug('Attempting to get complete cookie string (login process).');
const sessionId = await getSessionId();
if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId.");
@@ -195,24 +195,25 @@ async function getActivityDetailsRaw(activityId, cookies, maxRetries = 3, timeou
if (outerData && typeof outerData.d === 'string') {
const innerData = JSON.parse(outerData.d);
if (innerData.isError) {
console.warn(`API reported isError:true for activity ${activityId}.`);
logger.warn(`API reported isError:true for activity ${activityId}.`);
return null;
}
return response.data;
} else {
console.error(`Unexpected API response structure for activity ${activityId}.`);
logger.error(`Unexpected API response structure for activity ${activityId}.`);
}
} catch (error) {
if (error.response && (error.response.status === 403 || error.response.status === 401)) {
console.warn(`Authentication error (${error.response.status}) while fetching activity ${activityId}. Cookie may be invalid.`);
// Check if response status is in 4xx range (400-499) to trigger auth error
if (error.response && error.response.status >= 400 && error.response.status < 500) {
logger.warn(`Authentication error (${error.response.status}) while fetching activity ${activityId}. Cookie may be invalid.`);
throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
}
console.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
if (error.response) {
console.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
}
if (attempt === maxRetries - 1) {
console.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
throw error;
}
await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
@@ -241,27 +242,27 @@ export async function fetchActivityData(activityId, userName, userPwd, templateF
if (currentCookie) {
const isValid = await testCookieValidity(currentCookie);
if (!isValid) {
console.log("Cached cookie test failed or cookie expired. Clearing cache.");
logger.info("Cached cookie test failed or cookie expired. Clearing cache.");
await clearCookieCache();
currentCookie = null;
} else {
console.log("Using valid cached cookie.");
logger.info("Using valid cached cookie.");
}
}
if (!currentCookie) {
console.log(forceLogin ? "Forcing new login." : "No valid cached cookie found or cache bypassed. Attempting login...");
logger.info(forceLogin ? "Forcing new login." : "No valid cached cookie found or cache bypassed. Attempting login...");
try {
currentCookie = await getCompleteCookies(userName, userPwd, path.resolve(__dirname, templateFileName));
await saveCookieToCache(currentCookie);
} catch (loginError) {
console.error(`Login process failed: ${loginError.message}`);
logger.error(`Login process failed: ${loginError.message}`);
return null;
}
}
if (!currentCookie) {
console.error("Critical: No cookie available after login attempt. Cannot fetch activity data.");
logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data.");
return null;
}
@@ -271,32 +272,32 @@ export async function fetchActivityData(activityId, userName, userPwd, templateF
const parsedOuter = JSON.parse(rawActivityDetailsString);
return JSON.parse(parsedOuter.d);
}
console.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`);
logger.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`);
return null;
} catch (error) {
if (error instanceof AuthenticationError) {
console.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`);
logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`);
await clearCookieCache();
try {
console.log("Attempting re-login due to authentication failure...");
logger.info("Attempting re-login due to authentication failure...");
currentCookie = await getCompleteCookies(userName, userPwd, path.resolve(__dirname, templateFileName));
await saveCookieToCache(currentCookie);
console.log("Re-login successful. Retrying request for activity details once...");
logger.info("Re-login successful. Retrying request for activity details once...");
const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie);
if (rawActivityDetailsStringRetry) {
const parsedOuterRetry = JSON.parse(rawActivityDetailsStringRetry);
return JSON.parse(parsedOuterRetry.d);
}
console.warn(`Still no details for activity ${activityId} after re-login and retry.`);
logger.warn(`Still no details for activity ${activityId} after re-login and retry.`);
return null;
} catch (retryLoginOrFetchError) {
console.error(`Error during re-login or retry fetch for activity ${activityId}: ${retryLoginOrFetchError.message}`);
logger.error(`Error during re-login or retry fetch for activity ${activityId}: ${retryLoginOrFetchError.message}`);
return null;
}
} else {
console.error(`Failed to fetch activity data for ${activityId} due to non-authentication error: ${error.message}`);
logger.error(`Failed to fetch activity data for ${activityId} due to non-authentication error: ${error.message}`);
return null;
}
}

48
engage-api/struct-activity.mjs
View File

@@ -1,4 +1,5 @@
// struct-activity.mjs
import { logger } from '../utils/logger.mjs';
let clubSchema = {
academicYear: null,
@@ -84,21 +85,34 @@ async function applyFields(field, structuredActivityData) {
structuredActivityData.duration.isRecurringWeekly = true;
break;
default:
//console.log(`No matching case for field: fID=${field.fID}, fType=${field.fType}`);
logger.debug(`No matching case for field: fID=${field.fID}, fType=${field.fType}`);
break;
}
}
async function postProcess(structuredActivityData) {
structuredActivityData.description = structuredActivityData.description.replaceAll("<br/>","\n");
structuredActivityData.description = structuredActivityData.description.replaceAll("\u000B","\v");
if (structuredActivityData.name.search("Student-led") != -1) {
structuredActivityData.isStudentLed = true;
} else {
structuredActivityData.isStudentLed = false;
}
const grades = structuredActivityData.schedule.match(/G(\d+)-(\d+)/);
try {
let grades = structuredActivityData.schedule.match(/G(\d+)-(\d+)/) ||
structuredActivityData.schedule.match(/KG(\d+)-KG(\d+)/);
if (!grades || grades.length < 3) {
throw new Error('Invalid grade format in schedule');
}
structuredActivityData.grades.min = grades[1];
structuredActivityData.grades.max = grades[2];
} catch (error) {
logger.error(`Failed to parse grades: ${error.message}`);
structuredActivityData.grades.min = null;
structuredActivityData.grades.max = null;
}
}
export async function structActivityData(rawActivityData) {
@@ -110,35 +124,39 @@ export async function structActivityData(rawActivityData) {
for (let i = 0; i < rowObject.fields.length; i++) {
const field = rowObject.fields[i];
// Optimize: no fData, just skip
if (field.fData == null && field.fData == "") { continue; }
if (!field.fData) { continue; }
// Process hard cases first
if (field.fData == "Description") {
if (i + 1 < rowObject.fields.length) {
structuredActivityData.description = rowObject.fields[i + 1].fData;
}
continue;
} else if (field.fData == "Name To Appear On Reports") {
if (i + 1 < rowObject.fields.length) {
let staffForReports = rowObject.fields[i + 1].fData.split(", ");
structuredActivityData.staffForReports = staffForReports;
}
} else if (field.fData == "Upload Photo") {
if (i + 1 < rowObject.fields.length) {
structuredActivityData.photo = rowObject.fields[i + 1].fData;
}
} else if (field.fData == "Poor Weather Plan") {
if (i + 1 < rowObject.fields.length) {
structuredActivityData.poorWeatherPlan = rowObject.fields[i + 1].fData;
}
} else if (field.fData == "Activity Runs From") {
if (rowObject.fields[i + 4].fData == "Recurring Weekly") {
structuredActivityData.duration.isRecurringWeekly = true;
} else {
structuredActivityData.duration.isRecurringWeekly = false;
if (i + 4 < rowObject.fields.length) {
structuredActivityData.duration.isRecurringWeekly =
rowObject.fields[i + 4].fData == "Recurring Weekly";
}
} else if (field.fData == "Is Pre Sign-up") {
if (rowObject.fields[i + 1].fData == "") {
structuredActivityData.isPreSignup = false;
} else {
structuredActivityData.isPreSignup = true;
if (i + 1 < rowObject.fields.length) {
structuredActivityData.isPreSignup = rowObject.fields[i + 1].fData !== "";
}
} else if (field.fData == "Semester Cost") {
if (rowObject.fields[i + 1].fData == "") {
structuredActivityData.semesterCost = null;
} else {
structuredActivityData.semesterCost = rowObject.fields[i + 1].fData
if (i + 1 < rowObject.fields.length) {
structuredActivityData.semesterCost =
rowObject.fields[i + 1].fData === "" ? null : rowObject.fields[i + 1].fData;
}
} else {
// Pass any other easy cases to helper function

12
example.env
View File

@@ -3,3 +3,15 @@ API_PASSWORD=
PORT=3000
FIXED_STAFF_ACTIVITY_ID=7095
ALLOWED_ORIGINS=*
S3_ENDPOINT=
S3_BUCKET_NAME=
S3_ACCESS_KEY_ID=
S3_SECRET_ACCESS_KEY=
S3_REGION=
S3_PUBLIC_URL_PREFIX=files
REDIS_URL=redis://:dsas-cca@redis:6379
MAX_ACTIVITY_ID_SCAN=9999
CONCURRENT_API_CALLS=16
STAFF_UPDATE_INTERVAL_MINS=360
CLUB_UPDATE_INTERVAL_MINS=360
LOG_LEVEL=info # Example: 'debug', 'info', 'warn', 'error'

128
main.js
View File

@@ -1,128 +0,0 @@
// main.js
import express from 'express';
import dotenv from 'dotenv';
import cors from 'cors'; // Import the cors middleware
import { fetchActivityData } from './engage-api/get-activity.mjs';
import { structActivityData } from './engage-api/struct-activity.mjs';
import { structStaffData } from './engage-api/struct-staff.mjs';
// Load environment variables from .env file
dotenv.config();
// --- Configuration ---
const USERNAME = process.env.API_USERNAME;
const PASSWORD = process.env.API_PASSWORD;
const PORT = process.env.PORT || 3000;
const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID || '3350';
const allowedOriginsEnv = process.env.ALLOWED_ORIGINS || '*';
let corsOptions;
if (allowedOriginsEnv === '*') {
corsOptions = { origin: '*' };
} else {
// If ALLOWED_ORIGINS is a comma-separated list, split it into an array
const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim());
corsOptions = {
origin: function (origin, callback) {
// Allow requests with no origin (like mobile apps or curl requests)
if (!origin) return callback(null, true);
if (originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) {
callback(null, true);
} else {
callback(new Error('Not allowed by CORS'));
}
}
};
}
// --- Initialize Express App ---
const app = express();
// Apply CORS middleware globally FIRST
// This will add the 'Access-Control-Allow-Origin' header to all responses
app.use(cors(corsOptions));
// Middleware to parse JSON request bodies
app.use(express.json());
// --- API Endpoints ---
// GET Endpoint: Welcome message
app.get('/', (req, res) => {
res.send('Welcome to the DSAS CCA API!<br/><br/>\
API Endpoints:<br/>\
GET /v1/activity/:activityId (ID must be 1-4 digits)<br/>\
GET /v1/staffs');
});
// GET Endpoint: Fetch structured activity data by ID
app.get('/v1/activity/:activityId', async (req, res) => {
let { activityId } = req.params;
if (!/^\d{1,4}$/.test(activityId)) {
return res.status(400).json({
error: 'Invalid Activity ID format. Activity ID must be 1 to 4 digits (e.g., 1, 0001, 9999).'
});
}
if (!USERNAME || !PASSWORD) {
console.error('API username or password not configured. Check .env file or environment variables.');
return res.status(500).json({ error: 'Server configuration error.' });
}
console.log(`Workspaceing activity details for activity ID: ${activityId}`);
try {
const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
if (activityJson) {
const structuredActivity = await structActivityData(activityJson);
res.json(structuredActivity);
} else {
res.status(404).json({ error: `Could not retrieve details for activity ${activityId}.` });
}
} catch (error) {
console.error(`Error fetching activity ${activityId}:`, error);
res.status(500).json({ error: 'An internal server error occurred while fetching activity data.' });
}
});
// GET Endpoint: Fetch fixed structured staff data
app.get('/v1/staffs', async (req, res) => {
if (!USERNAME || !PASSWORD) {
console.error('API username or password not configured. Check .env file or environment variables.');
return res.status(500).json({ error: 'Server configuration error.' });
}
console.log(`Workspaceing staff details (using fixed activity ID: ${FIXED_STAFF_ACTIVITY_ID})`);
try {
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
const staffObject = Object.fromEntries(staffMap);
res.json(staffObject);
} else {
res.status(404).json({ error: `Could not retrieve base data using activity ID ${FIXED_STAFF_ACTIVITY_ID} to get staff details.` });
}
} catch (error) {
console.error(`Error fetching staff data (for fixed activity ID ${FIXED_STAFF_ACTIVITY_ID}):`, error);
res.status(500).json({ error: 'An internal server error occurred while fetching staff data.' });
}
});
// --- Start the Server ---
app.listen(PORT, () => {
console.log(`Server is running on http://localhost:${PORT}`);
console.log(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`);
console.log('API Endpoints:');
console.log(` GET /v1/activity/:activityId (ID must be 1-4 digits)`);
console.log(` GET /v1/staffs`);
if (!USERNAME || !PASSWORD) {
console.warn('Warning: API_USERNAME or API_PASSWORD is not set. Please configure them in your .env file or environment variables.');
}
});
// Graceful shutdown
process.on('SIGINT', () => {
console.log('Server shutting down...');
process.exit(0);
});

289
main.mjs Normal file
View File

@@ -0,0 +1,289 @@
// main.mjs
import express from 'express';
import dotenv from 'dotenv';
import cors from 'cors';
import { fetchActivityData } from './engage-api/get-activity.mjs';
import { structActivityData } from './engage-api/struct-activity.mjs';
import { structStaffData } from './engage-api/struct-staff.mjs';
import {
getActivityData,
setActivityData,
getStaffData,
setStaffData,
getRedisClient,
getAllActivityKeys, // Make sure this is imported
ACTIVITY_KEY_PREFIX // Import the prefix
} from './services/redis-service.mjs';
import { uploadImageFromBase64 } from './services/s3-service.mjs';
import { extractBase64Image } from './utils/image-processor.mjs';
import {
initializeClubCache,
updateStaleClubs,
initializeOrUpdateStaffCache,
cleanupOrphanedS3Images
} from './services/cache-manager.mjs';
import { logger } from './utils/logger.mjs';
dotenv.config();
const USERNAME = process.env.API_USERNAME;
const PASSWORD = process.env.API_PASSWORD;
const PORT = process.env.PORT || 3000;
const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID;
const allowedOriginsEnv = process.env.ALLOWED_ORIGINS || '*';
const CLUB_CHECK_INTERVAL_SECONDS = parseInt(process.env.CLUB_CHECK_INTERVAL_SECONDS || '300', 10);
const STAFF_CHECK_INTERVAL_SECONDS = parseInt(process.env.STAFF_CHECK_INTERVAL_SECONDS || '300', 10);
let corsOptions;
if (allowedOriginsEnv === '*') {
corsOptions = { origin: '*' };
} else {
const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim());
corsOptions = {
origin: function (origin, callback) {
if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) {
callback(null, true);
} else {
callback(new Error('Not allowed by CORS'));
}
}
};
}
const app = express();
app.use(cors(corsOptions));
app.use(express.json());
// Helper function to process activity data (fetch, struct, S3, cache) for API calls
async function fetchProcessAndStoreActivity(activityId) {
logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`);
const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
if (!activityJson) {
logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`);
const emptyData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' };
await setActivityData(activityId, emptyData);
return { data: emptyData, status: 404 };
}
let structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo && structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo);
if (imageInfo) {
const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
}
}
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
return { data: structuredActivity, status: 200 };
}
// --- API Endpoints ---
app.get('/', (req, res) => {
res.send('Welcome to the DSAS CCA API!<br/><br/>\
API Endpoints:<br/>\
GET /v1/activity/list<br/>\
GET /v1/activity/:activityId (ID must be 1-4 digits)<br/>\
GET /v1/staffs');
});
// NEW ENDPOINT: /v1/activity/list
app.get('/v1/activity/list', async (req, res) => {
try {
logger.info('Request received for /v1/activity/list');
const activityKeys = await getAllActivityKeys(); // From redis-service.mjs
const clubList = {};
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for list.');
return res.json({}); // Return empty object if no keys
}
// Fetch all activity data in parallel
// Note: This can be many individual GETs to Redis.
// For a very large number of keys, consider if this approach is too slow.
// However, Redis is fast, and Promise.all helps manage concurrency.
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach(activityData => {
// Check for a valid club:
// 1. activityData exists
// 2. It has an 'id' and a 'name'
// 3. It's not an error placeholder (no 'error' field)
// 4. It's not an intentionally empty record (no 'source: api-fetch-empty' field, or check if other essential fields are missing)
if (activityData &&
activityData.id &&
activityData.name && // Ensure name is present and not empty
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
clubList[activityData.id] = activityData.name;
}
});
logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs.`);
res.json(clubList);
} catch (error) {
logger.error('Error in /v1/activity/list endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating activity list.' });
}
});
app.get('/v1/activity/:activityId', async (req, res) => {
let { activityId } = req.params;
if (!/^\d{1,4}$/.test(activityId)) {
return res.status(400).json({ error: 'Invalid Activity ID format.' });
}
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' });
}
try {
let cachedActivity = await getActivityData(activityId);
const isValidCacheEntry = cachedActivity &&
!cachedActivity.error &&
Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0;
if (isValidCacheEntry) {
logger.info(`Cache HIT for activity ID: ${activityId}`);
cachedActivity.cache = "HIT";
return res.json(cachedActivity);
}
logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`);
const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId);
liveActivity.cache = "MISS";
if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) {
return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
}
res.status(status).json(liveActivity);
} catch (error) {
logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
}
});
app.get('/v1/staffs', async (req, res) => {
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' });
}
try {
let cachedStaffs = await getStaffData();
if (cachedStaffs && cachedStaffs.lastCheck) {
logger.info('Cache HIT for staffs.');
cachedStaffs.cache = "HIT";
return res.json(cachedStaffs);
}
logger.info('Cache MISS for staffs. Fetching from source.');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
let staffObject = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
staffObject.cache = "MISS";
await setStaffData(staffObject);
res.json(staffObject);
} else {
logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" });
}
} catch (error) {
logger.error('Error in /v1/staffs endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" });
}
});
// Function to perform background initialization and periodic tasks
async function performBackgroundTasks() {
logger.info('Starting background initialization tasks...');
try {
await initializeClubCache();
await initializeOrUpdateStaffCache(true);
await cleanupOrphanedS3Images();
logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000);
logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000);
logger.info('Background initialization and periodic task setup complete.');
} catch (error) {
logger.error('Error during background initialization tasks:', error);
}
}
// --- Start Server and Background Tasks ---
async function startServer() {
const redis = getRedisClient();
if (!redis) {
logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.');
process.exit(1);
}
try {
await redis.ping();
logger.info('Redis connection confirmed.');
app.listen(PORT, () => {
logger.info(`Server is running on http://localhost:${PORT}`);
logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`);
if (!USERNAME || !PASSWORD) {
logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.');
}
});
performBackgroundTasks().catch(error => {
logger.error('Unhandled error in performBackgroundTasks:', error);
});
} catch (err) {
logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err);
process.exit(1);
}
}
if (process.env.NODE_ENV !== 'test') {
startServer();
}
process.on('SIGINT', async () => {
logger.info('Server shutting down (SIGINT)...');
const redis = getRedisClient();
if (redis) {
await redis.quit();
logger.info('Redis connection closed.');
}
process.exit(0);
});
process.on('SIGTERM', async () => {
logger.info('Server shutting down (SIGTERM)...');
const redis = getRedisClient();
if (redis) {
await redis.quit();
logger.info('Redis connection closed.');
}
process.exit(0);
});
export { app };

10
package.json
View File

@@ -5,15 +5,19 @@
"main": "main.js",
"type": "module",
"scripts": {
"start": "node main.js",
"dev": "node --watch main.js"
"start": "node main.mjs",
"dev": "node --watch main.mjs"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.806.0",
"axios": "^1.9.0",
"cors": "^2.8.5",
"dotenv": "^16.5.0",
"express": "^5.1.0",
"node-fetch": "^3.3.2"
"ioredis": "^5.6.1",
"node-fetch": "^3.3.2",
"p-limit": "^6.2.0",
"uuid": "^11.1.0"
},
"packageManager": "pnpm@10.10.0+sha512.d615db246fe70f25dcfea6d8d73dee782ce23e2245e3c4f6f888249fb568149318637dca73c2c5c8ef2a4ca0d5657fb9567188bfab47f566d1ee6ce987815c39"
}

1277
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

209
services/cache-manager.mjs Normal file
View File

@@ -0,0 +1,209 @@
// services/cache-manager.mjs
import dotenv from 'dotenv';
import pLimit from 'p-limit';
import { fetchActivityData } from '../engage-api/get-activity.mjs';
import { structActivityData } from '../engage-api/struct-activity.mjs';
import { structStaffData } from '../engage-api/struct-staff.mjs';
import {
getActivityData,
setActivityData,
getStaffData,
setStaffData,
getAllActivityKeys as getAllRedisActivityKeys, // Renamed import for clarity
ACTIVITY_KEY_PREFIX
} from './redis-service.mjs';
import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service.mjs';
import { extractBase64Image } from '../utils/image-processor.mjs';
import { logger } from '../utils/logger.mjs';
dotenv.config();
const USERNAME = process.env.API_USERNAME;
const PASSWORD = process.env.API_PASSWORD;
const MAX_ACTIVITY_ID_SCAN = parseInt(process.env.MAX_ACTIVITY_ID_SCAN || '9999', 10);
const CONCURRENT_API_CALLS = parseInt(process.env.CONCURRENT_API_CALLS || '10', 10);
const CLUB_UPDATE_INTERVAL_MINS = parseInt(process.env.CLUB_UPDATE_INTERVAL_MINS || '60', 10);
const STAFF_UPDATE_INTERVAL_MINS = parseInt(process.env.STAFF_UPDATE_INTERVAL_MINS || '60', 10);
const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID;
const S3_IMAGE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').replace(/\/$/, ''); // Ensure no trailing slash
const limit = pLimit(CONCURRENT_API_CALLS);
async function processAndCacheActivity(activityId) {
logger.debug(`Processing activity ID: ${activityId}`);
try {
const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
let structuredActivity;
if (!activityJson) {
logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`);
structuredActivity = {};
} else {
structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo && structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo);
if (imageInfo) {
const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`Failed to upload image to S3 for activity ${activityId}. Base64 photo data will be removed or kept as is depending on structActivityData's behavior if upload fails.`);
// Potentially set photo to null or remove if upload is critical and fails
// structuredActivity.photo = null; // Example
}
}
}
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
// logger.info(`Successfully processed and cached activity ID: ${activityId}`); // Can be too verbose
return structuredActivity;
} catch (error) {
logger.error(`Error processing activity ID ${activityId}:`, error);
const errorData = { lastCheck: new Date().toISOString(), error: "Failed to fetch or process" };
await setActivityData(activityId, errorData);
return errorData;
}
}
export async function initializeClubCache() {
logger.info(`Starting initial club cache population up to ID ${MAX_ACTIVITY_ID_SCAN}...`);
const promises = [];
for (let i = 0; i <= MAX_ACTIVITY_ID_SCAN; i++) {
const activityId = String(i);
promises.push(limit(async () => {
const cachedData = await getActivityData(activityId);
if (!cachedData || Object.keys(cachedData).length === 0 || !cachedData.lastCheck || cachedData.error) {
logger.debug(`Initializing cache for activity ID: ${activityId}`);
await processAndCacheActivity(activityId);
}
}));
}
await Promise.all(promises);
logger.info('Initial club cache population finished.');
}
export async function updateStaleClubs() {
logger.info('Starting stale club check...');
const now = Date.now();
const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000;
const promises = [];
const activityKeys = await getAllRedisActivityKeys(); // More efficient than iterating 0-MAX_ID if many are empty
for (const key of activityKeys) {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
promises.push(limit(async () => {
const cachedData = await getActivityData(activityId); // Re-fetch to get latest before deciding
if (cachedData && cachedData.lastCheck) {
const lastCheckTime = new Date(cachedData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) { // Also update if previous fetch had an error
logger.info(`Activity ${activityId} is stale or had error. Updating...`);
await processAndCacheActivity(activityId);
}
} else if (!cachedData || Object.keys(cachedData).length === 0) {
// This case handles if a key was deleted or somehow became totally empty
logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`);
await processAndCacheActivity(activityId);
}
}));
}
// Optionally, iterate 0-MAX_ID_SCAN for any IDs not yet in Redis (newly created activities)
// This part can be heavy. Consider if getAllRedisActivityKeys is sufficient for "staleness".
// For truly new activities, they'd be picked up on direct API call or a less frequent full scan.
await Promise.all(promises);
logger.info('Stale club check finished.');
}
export async function initializeOrUpdateStaffCache(forceUpdate = false) {
logger.info('Starting staff cache check/update...');
try {
const cachedStaffData = await getStaffData();
const now = Date.now();
const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000;
let needsUpdate = forceUpdate;
if (!cachedStaffData || !cachedStaffData.lastCheck) {
needsUpdate = true;
} else {
const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs) {
needsUpdate = true;
}
}
if (needsUpdate) {
logger.info('Staff data needs update. Fetching...');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
const staffObject = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
await setStaffData(staffObject);
logger.info('Staff data updated and cached.');
} else {
logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
if(cachedStaffData && cachedStaffData.lastCheck){ // If old data exists, just update its lastCheck
cachedStaffData.lastCheck = new Date().toISOString();
await setStaffData(cachedStaffData);
}
}
} else {
logger.info('Staff data is up-to-date.');
}
} catch (error) {
logger.error('Error initializing or updating staff cache:', error);
}
}
export async function cleanupOrphanedS3Images() {
logger.info('Starting S3 orphan image cleanup...');
const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : ''; // Ensure trailing slash for prefix listing
try {
const referencedS3Urls = new Set();
const allActivityRedisKeys = await getAllRedisActivityKeys();
for (const redisKey of allActivityRedisKeys) {
const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length);
const activityData = await getActivityData(activityId); // Fetch by ID
if (activityData && typeof activityData.photo === 'string' && activityData.photo.startsWith('http')) { // Assuming S3 URLs start with http/https
// Check if the photo URL matches the expected S3 endpoint structure
if (activityData.photo.startsWith(process.env.S3_ENDPOINT)) {
referencedS3Urls.add(activityData.photo);
}
}
}
logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`);
const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix);
if (!s3ObjectKeys || s3ObjectKeys.length === 0) {
logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`);
return;
}
logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`);
const orphanedObjectKeys = [];
for (const objectKey of s3ObjectKeys) {
const s3Url = constructS3Url(objectKey); // Construct URL from key to compare
if (!referencedS3Urls.has(s3Url)) {
orphanedObjectKeys.push(objectKey);
}
}
if (orphanedObjectKeys.length > 0) {
logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`);
// orphanedObjectKeys.forEach(key => logger.debug(`Orphaned key: ${key}`)); // For verbose debugging
await deleteS3Objects(orphanedObjectKeys);
} else {
logger.info('No orphaned S3 images found after comparison.');
}
logger.info('S3 orphan image cleanup finished.');
} catch (error) {
logger.error('Error during S3 orphan image cleanup:', error);
}
}

135
services/redis-service.mjs Normal file
View File

@@ -0,0 +1,135 @@
// services/redis-service.mjs
import Redis from 'ioredis';
import dotenv from 'dotenv';
import { logger } from '../utils/logger.mjs';
dotenv.config();
const redisUrl = process.env.REDIS_URL;
let redisClient;
export const ACTIVITY_KEY_PREFIX = 'activity:'; // Exported for use in cache-manager
const STAFF_KEY = 'staffs:all';
try {
if (redisUrl) {
redisClient = new Redis(redisUrl);
redisClient.on('connect', () => {
logger.info('Connected to Redis successfully!');
});
redisClient.on('error', (err) => {
logger.error('Redis connection error:', err);
});
} else {
logger.error('REDIS_URL not defined. Redis client not initialized.');
}
} catch (error) {
logger.error('Failed to initialize Redis client:', error);
}
/**
* Gets activity data from Redis.
* @param {string} activityId
* @returns {Promise<object|null>} Parsed JSON object or null if not found/error.
*/
export async function getActivityData(activityId) {
if (!redisClient) {
logger.warn('Redis client not available, skipping getActivityData');
return null;
}
try {
const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`);
return data ? JSON.parse(data) : null;
} catch (err) {
logger.error(`Error getting activity ${activityId} from Redis:`, err);
return null;
}
}
/**
* Sets activity data in Redis.
* @param {string} activityId
* @param {object} data The activity data object.
* @returns {Promise<void>}
*/
export async function setActivityData(activityId, data) {
if (!redisClient) {
logger.warn('Redis client not available, skipping setActivityData');
return;
}
try {
await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data));
} catch (err) {
logger.error(`Error setting activity ${activityId} in Redis:`, err);
}
}
/**
* Gets staff data from Redis.
* @returns {Promise<object|null>} Parsed JSON object or null if not found/error.
*/
export async function getStaffData() {
if (!redisClient) {
logger.warn('Redis client not available, skipping getStaffData');
return null;
}
try {
const data = await redisClient.get(STAFF_KEY);
return data ? JSON.parse(data) : null;
} catch (err) {
logger.error('Error getting staff data from Redis:', err);
return null;
}
}
/**
* Sets staff data in Redis.
* @param {object} data The staff data object.
* @returns {Promise<void>}
*/
export async function setStaffData(data) {
if (!redisClient) {
logger.warn('Redis client not available, skipping setStaffData');
return;
}
try {
await redisClient.set(STAFF_KEY, JSON.stringify(data));
} catch (err) {
logger.error('Error setting staff data in Redis:', err);
}
}
/**
* Gets all activity keys from Redis.
* This can be resource-intensive on large datasets. Use with caution.
* @returns {Promise<string[]>} An array of keys.
*/
export async function getAllActivityKeys() {
if (!redisClient) {
logger.warn('Redis client not available, skipping getAllActivityKeys');
return [];
}
try {
// Using SCAN for better performance than KEYS on production Redis
let cursor = '0';
const keys = [];
do {
const [nextCursor, foundKeys] = await redisClient.scan(cursor, 'MATCH', `${ACTIVITY_KEY_PREFIX}*`, 'COUNT', '100');
keys.push(...foundKeys);
cursor = nextCursor;
} while (cursor !== '0');
logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`);
return keys;
} catch (err) {
logger.error('Error getting all activity keys from Redis using SCAN:', err);
return []; // Fallback or indicate error
}
}
export function getRedisClient() {
return redisClient;
}

186
services/s3-service.mjs Normal file
View File

@@ -0,0 +1,186 @@
// services/s3-service.mjs
import { S3Client, PutObjectCommand, ListObjectsV2Command, DeleteObjectsCommand } from '@aws-sdk/client-s3';
import { v4 as uuidv4 } from 'uuid';
import dotenv from 'dotenv';
import { logger } from '../utils/logger.mjs';
import { decodeBase64Image } from '../utils/image-processor.mjs';
dotenv.config();
const S3_ENDPOINT = process.env.S3_ENDPOINT;
const S3_REGION = process.env.S3_REGION;
const S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID;
const S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY;
const BUCKET_NAME = process.env.S3_BUCKET_NAME;
const PUBLIC_URL_FILE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').replace(/\/$/, ''); // Ensures no trailing slash
let s3Client;
if (S3_ENDPOINT && S3_REGION && S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {
s3Client = new S3Client({
endpoint: S3_ENDPOINT,
region: S3_REGION,
credentials: {
accessKeyId: S3_ACCESS_KEY_ID,
secretAccessKey: S3_SECRET_ACCESS_KEY,
},
forcePathStyle: true, // Important for MinIO and some S3-compatibles
});
} else {
logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.');
}
/**
* Uploads an image from a base64 string to S3.
* @param {string} base64Data The base64 content (without the data URI prefix).
* @param {string} originalFormat The image format (e.g., 'png', 'jpeg').
* @param {string} activityId The activity ID, used for naming.
* @returns {Promise<string|null>} The public URL of the uploaded image or null on error.
*/
export async function uploadImageFromBase64(base64Data, originalFormat, activityId) {
if (!s3Client) {
logger.warn('S3 client not configured. Cannot upload image.');
return null;
}
if (!base64Data || !originalFormat || !activityId) {
logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
return null;
}
try {
const imageBuffer = decodeBase64Image(base64Data);
// Ensure PUBLIC_URL_FILE_PREFIX is part of the key
const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.${originalFormat}`;
const params = {
Bucket: BUCKET_NAME,
Key: objectKey,
Body: imageBuffer,
ContentType: `image/${originalFormat}`,
ACL: 'public-read',
};
await s3Client.send(new PutObjectCommand(params));
const publicUrl = constructS3Url(objectKey);
logger.info(`Image uploaded to S3: ${publicUrl}`);
return publicUrl;
} catch (error) {
logger.error(`S3 Upload Error for activity ${activityId}:`, error);
return null;
}
}
/**
* Lists all objects in the S3 bucket under a specific prefix.
* @param {string} prefix The prefix to filter objects by (e.g., S3_PUBLIC_URL_PREFIX + '/').
* @returns {Promise<Array<string>>} A list of object keys.
*/
export async function listS3Objects(prefix) {
if (!s3Client) {
logger.warn('S3 client not configured. Cannot list objects.');
return [];
}
const objectKeys = [];
let isTruncated = true;
let continuationToken;
logger.debug(`Listing objects from S3 with prefix: "${prefix}"`);
const listCommandInput = { // Renamed to avoid conflict if command is redefined in loop
Bucket: BUCKET_NAME,
Prefix: prefix,
};
try {
while (isTruncated) {
if (continuationToken) {
listCommandInput.ContinuationToken = continuationToken;
}
const command = new ListObjectsV2Command(listCommandInput);
const { Contents, IsTruncated: NextIsTruncated, NextContinuationToken } = await s3Client.send(command);
if (Contents) {
Contents.forEach(item => {
if (item.Key && !item.Key.endsWith('/')) { // Ensure it's a file, not a pseudo-directory
objectKeys.push(item.Key);
}
});
}
isTruncated = NextIsTruncated;
continuationToken = NextContinuationToken;
}
logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
return objectKeys;
} catch (error) {
logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error);
return [];
}
}
/**
* Deletes multiple objects from S3.
* @param {Array<string>} objectKeysArray Array of object keys to delete.
* @returns {Promise<boolean>} True if successful or partially successful, false on major error.
*/
export async function deleteS3Objects(objectKeysArray) {
if (!s3Client) {
logger.warn('S3 client not configured. Cannot delete objects.');
return false;
}
if (!objectKeysArray || objectKeysArray.length === 0) {
logger.info('No objects to delete from S3.');
return true;
}
const MAX_DELETE_COUNT = 1000; // S3 API limit
let allDeletionsSuccessful = true;
for (let i = 0; i < objectKeysArray.length; i += MAX_DELETE_COUNT) {
const chunk = objectKeysArray.slice(i, i + MAX_DELETE_COUNT);
const deleteParams = {
Bucket: BUCKET_NAME,
Delete: {
Objects: chunk.map(key => ({ Key: key })),
Quiet: false, // We want error details
},
};
try {
const command = new DeleteObjectsCommand(deleteParams);
const output = await s3Client.send(command);
if (output.Errors && output.Errors.length > 0) {
allDeletionsSuccessful = false;
output.Errors.forEach(err => {
logger.error(`S3 Delete Error for key ${err.Key}: ${err.Message}`);
});
}
if (output.Deleted && output.Deleted.length > 0) {
logger.info(`Successfully submitted deletion for ${output.Deleted.length} objects from S3 chunk (some might have failed, check individual errors).`);
}
} catch (error) {
logger.error('S3 DeleteObjects Command Error for a chunk:', error);
allDeletionsSuccessful = false;
}
}
if (allDeletionsSuccessful && objectKeysArray.length > 0) {
logger.info(`Finished S3 deletion request for ${objectKeysArray.length} keys.`);
} else if (objectKeysArray.length > 0) {
logger.warn(`S3 deletion request for ${objectKeysArray.length} keys completed with some errors.`);
}
return allDeletionsSuccessful;
}
/**
* Constructs the public S3 URL for an object key.
* @param {string} objectKey The key of the object in S3.
* @returns {string} The full public URL.
*/
export function constructS3Url(objectKey) {
// Ensure S3_ENDPOINT does not end with a slash
const s3Base = S3_ENDPOINT.replace(/\/$/, '');
// Ensure BUCKET_NAME does not start or end with a slash
const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, '');
// Ensure objectKey does not start with a slash
const key = objectKey.replace(/^\//, '');
return `${s3Base}/${bucket}/${key}`;
}

43
utils/image-processor.mjs Normal file
View File

@@ -0,0 +1,43 @@
// utils/image-processor.mjs
import { logger } from './logger.mjs'; // Using the logger
/**
* Extracts base64 content and format from a data URL string.
* E.g., "data:image/jpeg;base64,xxxxxxxxxxxxxxx"
* @param {string} dataUrl The full data URL string.
* @returns {object|null} An object { base64Content: string, format: string } or null if not found.
*/
export function extractBase64Image(dataUrl) {
if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) {
return null;
}
const markers = [
{ prefix: "data:image/png;base64,", format: "png" },
{ prefix: "data:image/jpeg;base64,", format: "jpeg" },
{ prefix: "data:image/jpg;base64,", format: "jpg" },
{ prefix: "data:image/gif;base64,", format: "gif" },
{ prefix: "data:image/svg+xml;base64,", format: "svg" }, // svg+xml -> svg
{ prefix: "data:image/webp;base64,", format: "webp" }
];
for (const marker of markers) {
if (dataUrl.startsWith(marker.prefix)) {
const base64Content = dataUrl.substring(marker.prefix.length);
logger.debug(`Found image of format: ${marker.format}`);
return { base64Content, format: marker.format };
}
}
logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "...");
return null;
}
/**
* Decodes a base64 string to a Buffer.
* @param {string} base64String The base64 encoded string (without the data URI prefix).
* @returns {Buffer}
*/
export function decodeBase64Image(base64String) {
return Buffer.from(base64String, 'base64');
}

30
utils/logger.mjs Normal file
View File

@@ -0,0 +1,30 @@
// utils/logger.mjs
import dotenv from 'dotenv';
dotenv.config(); // Ensure .env variables are loaded
const LOG_LEVEL = process.env.LOG_LEVEL || 'info'; // Example: 'debug', 'info', 'warn', 'error'
const levels = {
error: 0,
warn: 1,
info: 2,
debug: 3,
};
const currentLevel = levels[LOG_LEVEL.toLowerCase()] ?? levels.info;
const log = (level, ...args) => {
if (levels[level] <= currentLevel) {
const timestamp = new Date().toISOString();
console[level](`[${timestamp}] [${level.toUpperCase()}]`, ...args);
}
};
export const logger = {
error: (...args) => log('error', ...args),
warn: (...args) => log('warn', ...args),
info: (...args) => log('info', ...args),
debug: (...args) => log('debug', ...args),
};
export default logger;