JamesFlare/dsas-cca-backend-bun
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: JamesFlare:2100bd04cad5ff132557b1cd0a4f1a6aea643044
Branches Tags
JamesFlare:master
..
compare: JamesFlare:7ba5f8f00fef6a7ced76acbcf7cb77b53d4cc116
Branches Tags
JamesFlare:master

2 Commits
2100bd04ca ... 7ba5f8f00f

Author SHA1 Message Date
JamesFlare1212
7ba5f8f00f feat: skip duplicate image on remote 2025-05-12 23:46:25 -04:00
JamesFlare1212
8598571f72 improve: code structure 2025-05-12 21:45:57 -04:00
10 changed files with 859 additions and 878 deletions
Expand all files Collapse all files

3
bun.lock
View File

@@ -6,6 +6,7 @@
"dependencies": { "dependencies": {
"axios": "^1.9.0", "axios": "^1.9.0",
"cors": "^2.8.5", "cors": "^2.8.5",
"crypto": "^1.0.1",
"dotenv": "^16.5.0", "dotenv": "^16.5.0",
"express": "^5.1.0", "express": "^5.1.0",
"p-limit": "^6.2.0", "p-limit": "^6.2.0",
@@ -104,6 +105,8 @@
"cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="], "cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="],
"crypto": ["crypto@1.0.1", "", {}, "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="],
"debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="], "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],

110
engage-api/get-activity.ts
View File

@@ -1,21 +1,21 @@
// ./engage-api/get-activity.ts // engage-api/get-activity.ts
import axios from 'axios'; import axios from 'axios';
import { readFile, writeFile, unlink } from 'fs/promises'; import { readFile,writeFile,unlink } from 'fs/promises';
import { resolve } from 'path'; import { resolve } from 'path';
import { logger } from '../utils/logger'; import { logger } from '../utils/logger';
// Define interfaces for our data structures // Define interfaces for our data structures
interface ActivityResponse { interface ActivityResponse {
d: string; d: string;
isError?: boolean; isError ? : boolean;
[key: string]: any; [key: string]: any;
} }
// --- Custom Error for Authentication --- // Custom Error for Authentication
class AuthenticationError extends Error { class AuthenticationError extends Error {
status: number; status: number;
constructor(message: string = "Authentication failed, cookie may be invalid.", status?: number) { constructor(message: string = "Authentication failed, cookie may be invalid.", status ? : number) {
super(message); super(message);
this.name = "AuthenticationError"; this.name = "AuthenticationError";
this.status = status || 0; this.status = status || 0;
@@ -26,8 +26,8 @@ class AuthenticationError extends Error {
const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt'); const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt');
let _inMemoryCookie: string | null = null; let _inMemoryCookie: string | null = null;
// --- Cookie Cache Helper Functions --- // Cookie Cache Helper Functions
async function loadCachedCookie(): Promise<string | null> { async function loadCachedCookie(): Promise < string | null > {
if (_inMemoryCookie) { if (_inMemoryCookie) {
logger.debug("Using in-memory cached cookie."); logger.debug("Using in-memory cached cookie.");
return _inMemoryCookie; return _inMemoryCookie;
@@ -49,7 +49,7 @@ async function loadCachedCookie(): Promise<string | null> {
return null; return null;
} }
async function saveCookieToCache(cookieString: string): Promise<void> { async function saveCookieToCache(cookieString: string): Promise < void > {
if (!cookieString) { if (!cookieString) {
logger.warn("Attempted to save an empty or null cookie. Aborting save."); logger.warn("Attempted to save an empty or null cookie. Aborting save.");
return; return;
@@ -63,7 +63,7 @@ async function saveCookieToCache(cookieString: string): Promise<void> {
} }
} }
async function clearCookieCache(): Promise<void> { async function clearCookieCache(): Promise < void > {
_inMemoryCookie = null; _inMemoryCookie = null;
try { try {
await unlink(COOKIE_FILE_PATH); await unlink(COOKIE_FILE_PATH);
@@ -77,13 +77,13 @@ async function clearCookieCache(): Promise<void> {
} }
} }
async function testCookieValidity(cookieString: string): Promise<boolean> { async function testCookieValidity(cookieString: string): Promise < boolean > {
if (!cookieString) return false; if (!cookieString) return false;
logger.debug("Testing cookie validity..."); logger.debug("Testing cookie validity...");
const MAX_RETRIES = 3; const MAX_RETRIES = 3;
let attempt = 0; let attempt = 0;
while (attempt < MAX_RETRIES) { while (attempt < MAX_RETRIES) {
try { try {
attempt++; attempt++;
@@ -93,11 +93,16 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
'Cookie': cookieString, 'Cookie': cookieString,
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)', 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
}; };
const payload = { "activityID": "3350" }; const payload = {
"activityID": "3350"
};
logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`); logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`);
await axios.post(url, payload, { headers, timeout: 20000 }); await axios.post(url, payload, {
headers,
timeout: 20000
});
logger.debug("Cookie test successful (API responded 2xx). Cookie is valid."); logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
return true; return true;
} catch (error: any) { } catch (error: any) {
@@ -107,7 +112,7 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
} else { } else {
logger.warn(`Network/other error: ${error.message}`); logger.warn(`Network/other error: ${error.message}`);
} }
if (attempt >= MAX_RETRIES) { if (attempt >= MAX_RETRIES) {
logger.warn("Max retries reached. Cookie is likely invalid or expired."); logger.warn("Max retries reached. Cookie is likely invalid or expired.");
return false; return false;
@@ -117,12 +122,14 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
return false; return false;
} }
// --- Core API Interaction Functions --- // Core API Interaction Functions
async function getSessionId(): Promise<string | null> { async function getSessionId(): Promise < string | null > {
const url = 'https://engage.nkcswx.cn/Login.aspx'; const url = 'https://engage.nkcswx.cn/Login.aspx';
try { try {
const response = await axios.get(url, { const response = await axios.get(url, {
headers: { 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)' } headers: {
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)'
}
}); });
const setCookieHeader = response.headers['set-cookie']; const setCookieHeader = response.headers['set-cookie'];
if (setCookieHeader && setCookieHeader.length > 0) { if (setCookieHeader && setCookieHeader.length > 0) {
@@ -141,7 +148,7 @@ async function getSessionId(): Promise<string | null> {
} }
} }
async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise<string | null> { async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise < string | null > {
const url = 'https://engage.nkcswx.cn/Login.aspx'; const url = 'https://engage.nkcswx.cn/Login.aspx';
try { try {
let templateData = await readFile(templateFilePath, 'utf8'); let templateData = await readFile(templateFilePath, 'utf8');
@@ -149,14 +156,14 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
.replace('{{USERNAME}}', userName) .replace('{{USERNAME}}', userName)
.replace('{{PASSWORD}}', userPwd); .replace('{{PASSWORD}}', userPwd);
const headers = { const headers = {
'Content-Type': 'application/x-www-form-urlencoded', 'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': sessionId, 'Cookie': sessionId,
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)', 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
'Referer': 'https://engage.nkcswx.cn/Login.aspx' 'Referer': 'https://engage.nkcswx.cn/Login.aspx'
}; };
logger.debug('Getting .ASPXFORMSAUTH'); logger.debug('Getting .ASPXFORMSAUTH');
const response = await axios.post(url, postData, { const response = await axios.post(url, postData, {
headers, headers,
maxRedirects: 0, maxRedirects: 0,
validateStatus: (status) => status >= 200 && status < 400 validateStatus: (status) => status >= 200 && status < 400
}); });
@@ -167,10 +174,10 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
if (aspxAuthCookies.length > 0) { if (aspxAuthCookies.length > 0) {
for (let i = aspxAuthCookies.length - 1; i >= 0; i--) { for (let i = aspxAuthCookies.length - 1; i >= 0; i--) {
const cookieCandidateParts = aspxAuthCookies[i].split(';'); const cookieCandidateParts = aspxAuthCookies[i].split(';');
if (cookieCandidateParts.length > 0 && cookieCandidateParts[0] !== undefined) { // Explicit check if (cookieCandidateParts.length > 0 && cookieCandidateParts[0] !== undefined) { // Explicit check
const firstPart = cookieCandidateParts[0].trim(); const firstPart = cookieCandidateParts[0].trim();
if (firstPart.length > '.ASPXFORMSAUTH='.length && firstPart.substring('.ASPXFORMSAUTH='.length).length > 0) { if (firstPart.length > '.ASPXFORMSAUTH='.length && firstPart.substring('.ASPXFORMSAUTH='.length).length > 0) {
formsAuthCookieValue = firstPart; formsAuthCookieValue = firstPart;
break; break;
} }
} }
@@ -191,37 +198,39 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
} }
} }
async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise<string> { async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise < string > {
logger.debug('Attempting to get complete cookie string (login process).'); logger.debug('Attempting to get complete cookie string (login process).');
const sessionId = await getSessionId(); const sessionId = await getSessionId();
if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId."); if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId.");
const msAuth = await getMSAUTH(sessionId, userName, userPwd, templateFilePath); const msAuth = await getMSAUTH(sessionId, userName, userPwd, templateFilePath);
if (!msAuth) throw new Error("Login failed: Could not obtain .ASPXFORMSAUTH cookie."); if (!msAuth) throw new Error("Login failed: Could not obtain .ASPXFORMSAUTH cookie.");
return `${sessionId}; ${msAuth}`; return `${sessionId}; ${msAuth}`;
} }
async function getActivityDetailsRaw( async function getActivityDetailsRaw(
activityId: string, activityId: string,
cookies: string, cookies: string,
maxRetries: number = 3, maxRetries: number = 3,
timeoutMilliseconds: number = 20000 timeoutMilliseconds: number = 20000
): Promise<string | null> { ): Promise < string | null > {
const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails'; const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
const headers = { const headers = {
'Content-Type': 'application/json; charset=UTF-8', 'Content-Type': 'application/json; charset=UTF-8',
'Cookie': cookies, 'Cookie': cookies,
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)', 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
'X-Requested-With': 'XMLHttpRequest' 'X-Requested-With': 'XMLHttpRequest'
}; };
const payload = { "activityID": String(activityId) }; const payload = {
"activityID": String(activityId)
};
for (let attempt = 0; attempt < maxRetries; attempt++) { for (let attempt = 0; attempt < maxRetries; attempt++) {
try { try {
const response = await axios.post(url, payload, { const response = await axios.post(url, payload, {
headers, headers,
timeout: timeoutMilliseconds, timeout: timeoutMilliseconds,
responseType: 'text' responseType: 'text'
}); });
const outerData = JSON.parse(response.data); const outerData = JSON.parse(response.data);
@@ -242,12 +251,13 @@ async function getActivityDetailsRaw(
throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status); throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
} }
logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`); logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
if (error.response) { if (error.response) {
logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`); logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
} }
if (attempt === maxRetries - 1) { if (attempt === maxRetries - 1) {
logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`); logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
throw error; throw error;
} }
await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1))); await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
} }
@@ -265,14 +275,14 @@ async function getActivityDetailsRaw(
* @returns The parsed JSON object of activity details, or null on failure. * @returns The parsed JSON object of activity details, or null on failure.
*/ */
export async function fetchActivityData( export async function fetchActivityData(
activityId: string, activityId: string,
userName: string, userName: string,
userPwd: string, userPwd: string,
templateFileName: string = "login_template.txt", templateFileName: string = "login_template.txt",
forceLogin: boolean = false forceLogin: boolean = false
): Promise<any | null> { ): Promise < any | null > {
let currentCookie = forceLogin ? null : await loadCachedCookie(); let currentCookie = forceLogin ? null : await loadCachedCookie();
if (forceLogin && currentCookie) { if (forceLogin && currentCookie) {
await clearCookieCache(); await clearCookieCache();
currentCookie = null; currentCookie = null;
@@ -304,7 +314,7 @@ export async function fetchActivityData(
logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data."); logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data.");
return null; return null;
} }
try { try {
const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie); const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie);
if (rawActivityDetailsString) { if (rawActivityDetailsString) {
@@ -316,13 +326,13 @@ export async function fetchActivityData(
} catch (error) { } catch (error) {
if (error instanceof AuthenticationError) { if (error instanceof AuthenticationError) {
logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`); logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`);
await clearCookieCache(); await clearCookieCache();
try { try {
logger.info("Attempting re-login due to authentication failure..."); logger.info("Attempting re-login due to authentication failure...");
currentCookie = await getCompleteCookies(userName, userPwd, resolve(import.meta.dir, templateFileName)); currentCookie = await getCompleteCookies(userName, userPwd, resolve(import.meta.dir, templateFileName));
await saveCookieToCache(currentCookie); await saveCookieToCache(currentCookie);
logger.info("Re-login successful. Retrying request for activity details once..."); logger.info("Re-login successful. Retrying request for activity details once...");
const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie); const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie);
if (rawActivityDetailsStringRetry) { if (rawActivityDetailsStringRetry) {
@@ -343,4 +353,4 @@ export async function fetchActivityData(
} }
// Optionally // Optionally
export { clearCookieCache, testCookieValidity }; //export { clearCookieCache,testCookieValidity };

2
engage-api/struct-activity.ts
View File

@@ -254,4 +254,4 @@ export async function structActivityData(rawActivityData: RawActivityData): Prom
} }
await postProcess(structuredActivityData); await postProcess(structuredActivityData);
return structuredActivityData; return structuredActivityData;
} }

730
index.ts
View File

@@ -6,51 +6,41 @@ import { fetchActivityData } from './engage-api/get-activity';
import { structActivityData } from './engage-api/struct-activity'; import { structActivityData } from './engage-api/struct-activity';
import { structStaffData } from './engage-api/struct-staff'; import { structStaffData } from './engage-api/struct-staff';
import { import {
getActivityData, getActivityData,
setActivityData, setActivityData,
getStaffData, getStaffData,
setStaffData, setStaffData,
getRedisClient, getRedisClient,
getAllActivityKeys, getAllActivityKeys,
ACTIVITY_KEY_PREFIX, ACTIVITY_KEY_PREFIX,
closeRedisConnection closeRedisConnection
} from './services/redis-service'; } from './services/redis-service';
import { uploadImageFromBase64 } from './services/s3-service'; import { uploadImageFromBase64 } from './services/s3-service';
import { extractBase64Image } from './utils/image-processor'; import { extractBase64Image } from './utils/image-processor';
import { import {
initializeClubCache, initializeClubCache,
updateStaleClubs, updateStaleClubs,
initializeOrUpdateStaffCache, initializeOrUpdateStaffCache,
cleanupOrphanedS3Images cleanupOrphanedS3Images
} from './services/cache-manager'; } from './services/cache-manager';
import { logger } from './utils/logger'; import { logger } from './utils/logger';
import type { ActivityData } from './models/activity'
// Define interfaces for our data structures // Define interfaces for our data structures
interface ActivityData {
id?: string;
name?: string;
photo?: string;
lastCheck?: string;
source?: string;
error?: string;
cache?: string;
[key: string]: any;
}
interface StaffData { interface StaffData {
lastCheck?: string; lastCheck?: string;
cache?: string; cache?: string;
[key: string]: any; [key: string]: any;
} }
interface ImageInfo { interface ImageInfo {
base64Content: string; base64Content: string;
format: string; format: string;
} }
interface ProcessedActivityResult { interface ProcessedActivityResult {
data: ActivityData; data: ActivityData;
status: number; status: number;
} }
config(); config();
@@ -65,23 +55,23 @@ const STAFF_CHECK_INTERVAL_SECONDS = parseInt(process.env.STAFF_CHECK_INTERVAL_S
// CORS configuration // CORS configuration
type CorsOptions = { type CorsOptions = {
origin: string | string[] | ((origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) => void); origin: string | string[] | ((origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) => void);
}; };
let corsOptions: CorsOptions; let corsOptions: CorsOptions;
if (allowedOriginsEnv === '*') { if (allowedOriginsEnv === '*') {
corsOptions = { origin: '*' }; corsOptions = { origin: '*' };
} else { } else {
const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim()); const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim());
corsOptions = { corsOptions = {
origin: function (origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) { origin: function (origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) {
if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) { if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) {
callback(null, true); callback(null, true);
} else { } else {
callback(new Error('Not allowed by CORS')); callback(new Error('Not allowed by CORS'));
} }
} }
}; };
} }
const app = express(); const app = express();
@@ -90,404 +80,386 @@ app.use(express.json());
// Helper function to process activity data (fetch, struct, S3, cache) for API calls // Helper function to process activity data (fetch, struct, S3, cache) for API calls
async function fetchProcessAndStoreActivity(activityId: string): Promise<ProcessedActivityResult> { async function fetchProcessAndStoreActivity(activityId: string): Promise<ProcessedActivityResult> {
logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`); logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`);
const activityJson = await fetchActivityData(activityId, USERNAME as string, PASSWORD as string); const activityJson = await fetchActivityData(activityId, USERNAME as string, PASSWORD as string);
if (!activityJson) { if (!activityJson) {
logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`); logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`);
const emptyData: ActivityData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' }; const emptyData: ActivityData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' };
await setActivityData(activityId, emptyData); await setActivityData(activityId, emptyData);
return { data: emptyData, status: 404 }; return { data: emptyData, status: 404 };
} }
let structuredActivity = await structActivityData(activityJson); let structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo && if (structuredActivity && structuredActivity.photo &&
typeof structuredActivity.photo === 'string' && typeof structuredActivity.photo === 'string' &&
structuredActivity.photo.startsWith('data:image')) { structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo) as ImageInfo | null; const imageInfo = extractBase64Image(structuredActivity.photo) as ImageInfo | null;
if (imageInfo) { if (imageInfo) {
const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId); const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId);
if (s3Url) { if (s3Url) {
structuredActivity.photo = s3Url; structuredActivity.photo = s3Url;
} else { } else {
logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`); logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
} }
}
} }
structuredActivity.lastCheck = new Date().toISOString(); }
await setActivityData(activityId, structuredActivity); structuredActivity.lastCheck = new Date().toISOString();
return { data: structuredActivity, status: 200 }; await setActivityData(activityId, structuredActivity);
return { data: structuredActivity, status: 200 };
} }
// --- API Endpoints --- // --- API Endpoints ---
app.get('/', (_req: Request, res: Response) => { app.get('/', (_req: Request, res: Response) => {
res.send('Welcome to the DSAS CCA API!<br/>\ res.send('Welcome to the DSAS CCA API!<br/>\
GET /v1/activity/list<br/>\ GET /v1/activity/list<br/>\
GET /v1/activity/list?category=<br/>\ GET /v1/activity/list?category=<br/>\
GET /v1/activity/list?academicYear=<br/>\ GET /v1/activity/list?academicYear=<br/>\
GET /v1/activity/list?grade=<br/>\ GET /v1/activity/list?grade=<br/>\
GET /v1/activity/category<br/>\ GET /v1/activity/category<br/>\
GET /v1/activity/academicYear<br/>\ GET /v1/activity/academicYear<br/>\
GET /v1/activity/:activityId<br/>\ GET /v1/activity/:activityId<br/>\
GET /v1/staffs'); GET /v1/staffs');
}); });
// Activity list endpoint with filtering capabilities // Activity list endpoint with filtering capabilities
app.get('/v1/activity/list', async (req: Request, res: Response) => { app.get('/v1/activity/list', async (req: Request, res: Response) => {
try { try {
const category = req.query.category as string | undefined; const category = req.query.category as string | undefined;
const academicYear = req.query.academicYear as string | undefined; const academicYear = req.query.academicYear as string | undefined;
const grade = req.query.grade as string | undefined; const grade = req.query.grade as string | undefined;
// Validate academicYear format if provided (YYYY/YYYY)
// Validate academicYear format if provided (YYYY/YYYY) if (academicYear !== undefined) {
if (academicYear !== undefined) { const academicYearRegex = /^\d{4}\/\d{4}$/;
const academicYearRegex = /^\d{4}\/\d{4}$/; if (!academicYearRegex.test(academicYear)) {
if (!academicYearRegex.test(academicYear)) { return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' });
return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' }); }
}
}
// Validate grade if provided
let validGrade: number | null = null;
if (grade !== undefined) {
const parsedGrade = parseInt(grade, 10);
if (!isNaN(parsedGrade) && parsedGrade > 0 && parsedGrade <= 12) {
validGrade = parsedGrade;
} else {
return res.status(400).json({ error: 'Invalid grade parameter. Must be a number between 1 and 12.' });
}
}
logger.info(`Request received for /v1/activity/list with filters: ${JSON.stringify({category, academicYear, grade: validGrade})}`);
const activityKeys = await getAllActivityKeys();
const clubList: Record<string, {name: string, photo: string}> = {};
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for list.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
// First pass: collect all available categories for validation
const availableCategories = new Set<string>();
const availableAcademicYears = new Set<string>();
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (activityData.category) {
availableCategories.add(activityData.category);
}
if (activityData.academicYear) {
availableAcademicYears.add(activityData.academicYear);
}
}
});
// Validate category against available categories
if (category && !availableCategories.has(category)) {
return res.status(400).json({
error: 'Invalid category parameter. Category not found.',
availableCategories: Array.from(availableCategories)
});
}
// Validate academicYear against available years
if (academicYear && !availableAcademicYears.has(academicYear)) {
return res.status(400).json({
error: 'Invalid academicYear parameter. Academic year not found.',
availableAcademicYears: Array.from(availableAcademicYears)
});
}
// Apply filters and collect club data
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.id &&
activityData.name &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
// Check if it matches category filter if provided
if (category && activityData.category !== category) {
return; // Skip this activity
}
// Check if it matches academicYear filter if provided
if (academicYear && activityData.academicYear !== academicYear) {
return; // Skip this activity
}
// Check if it matches grade filter if provided
if (validGrade !== null) {
// Skip if grades are null
if (!activityData.grades ||
activityData.grades.min === null ||
activityData.grades.max === null) {
return; // Skip this activity
}
const minGrade = parseInt(activityData.grades.min, 10);
const maxGrade = parseInt(activityData.grades.max, 10);
// Skip if grade is out of range or if parsing fails
if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) {
return; // Skip this activity
}
}
// Add to result object with name and photo
clubList[activityData.id] = {
name: activityData.name,
photo: activityData.photo || ""
};
}
});
logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`);
res.json(clubList);
} catch (error) {
logger.error('Error in /v1/activity/list endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating activity list.' });
} }
// Validate grade if provided
let validGrade: number | null = null;
if (grade !== undefined) {
const parsedGrade = parseInt(grade, 10);
if (!isNaN(parsedGrade) && parsedGrade > 0 && parsedGrade <= 12) {
validGrade = parsedGrade;
} else {
return res.status(400).json({ error: 'Invalid grade parameter. Must be a number between 1 and 12.' });
}
}
logger.info(`Request received for /v1/activity/list with filters: ${JSON.stringify({category, academicYear, grade: validGrade})}`);
const activityKeys = await getAllActivityKeys();
const clubList: Record<string, {name: string, photo: string}> = {};
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for list.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
// First pass: collect all available categories for validation
const availableCategories = new Set<string>();
const availableAcademicYears = new Set<string>();
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (activityData.category) {
availableCategories.add(activityData.category);
}
if (activityData.academicYear) {
availableAcademicYears.add(activityData.academicYear);
}
}
});
// Validate category against available categories
if (category && !availableCategories.has(category)) {
return res.status(400).json({
error: 'Invalid category parameter. Category not found.',
availableCategories: Array.from(availableCategories)
});
}
// Validate academicYear against available years
if (academicYear && !availableAcademicYears.has(academicYear)) {
return res.status(400).json({
error: 'Invalid academicYear parameter. Academic year not found.',
availableAcademicYears: Array.from(availableAcademicYears)
});
}
// Apply filters and collect club data
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.id &&
activityData.name &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
// Check if it matches category filter if provided
if (category && activityData.category !== category) {
return; // Skip this activity
}
// Check if it matches academicYear filter if provided
if (academicYear && activityData.academicYear !== academicYear) {
return; // Skip this activity
}
// Check if it matches grade filter if provided
if (validGrade !== null) {
// Skip if grades are null
if (!activityData.grades ||
activityData.grades.min === null ||
activityData.grades.max === null) {
return; // Skip this activity
}
const minGrade = parseInt(activityData.grades.min, 10);
const maxGrade = parseInt(activityData.grades.max, 10);
// Skip if grade is out of range or if parsing fails
if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) {
return; // Skip this activity
}
}
// Add to result object with name and photo
clubList[activityData.id] = {
name: activityData.name,
photo: activityData.photo || ""
};
}
});
logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`);
res.json(clubList);
} catch (error) {
logger.error('Error in /v1/activity/list endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating activity list.' });
}
}); });
// Category endpoint // Category endpoint
app.get('/v1/activity/category', async (_req: Request, res: Response) => { app.get('/v1/activity/category', async (_req: Request, res: Response) => {
try { try {
logger.info('Request received for /v1/activity/category'); logger.info('Request received for /v1/activity/category');
const activityKeys = await getAllActivityKeys(); const activityKeys = await getAllActivityKeys();
const categoryMap: Record<string, number> = {}; const categoryMap: Record<string, number> = {};
if (!activityKeys || activityKeys.length === 0) { if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for categories.'); logger.info('No activity keys found in Redis for categories.');
return res.json({}); return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.category &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (categoryMap[activityData.category]) {
categoryMap[activityData.category] = (categoryMap[activityData.category] ?? 0) + 1;
} else {
categoryMap[activityData.category] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`);
res.json(categoryMap);
} catch (error) {
logger.error('Error in /v1/activity/category endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating category list.' });
} }
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.category &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (categoryMap[activityData.category]) {
categoryMap[activityData.category] = (categoryMap[activityData.category] ?? 0) + 1;
} else {
categoryMap[activityData.category] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`);
res.json(categoryMap);
} catch (error) {
logger.error('Error in /v1/activity/category endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating category list.' });
}
}); });
// Academic Year endpoint // Academic Year endpoint
app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => { app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {
try { try {
logger.info('Request received for /v1/activity/academicYear'); logger.info('Request received for /v1/activity/academicYear');
const activityKeys = await getAllActivityKeys(); const activityKeys = await getAllActivityKeys();
const academicYearMap: Record<string, number> = {}; const academicYearMap: Record<string, number> = {};
if (!activityKeys || activityKeys.length === 0) { if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for academic years.'); logger.info('No activity keys found in Redis for academic years.');
return res.json({}); return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.academicYear &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (academicYearMap[activityData.academicYear]) {
academicYearMap[activityData.academicYear] = (academicYearMap[activityData.academicYear] ?? 0) + 1;
} else {
academicYearMap[activityData.academicYear] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`);
res.json(academicYearMap);
} catch (error) {
logger.error('Error in /v1/activity/academicYear endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' });
} }
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.academicYear &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (academicYearMap[activityData.academicYear]) {
academicYearMap[activityData.academicYear] = (academicYearMap[activityData.academicYear] ?? 0) + 1;
} else {
academicYearMap[activityData.academicYear] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`);
res.json(academicYearMap);
} catch (error) {
logger.error('Error in /v1/activity/academicYear endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' });
}
}); });
// Single activity endpoint // Single activity endpoint
app.get('/v1/activity/:activityId', async (req: Request, res: Response) => { app.get('/v1/activity/:activityId', async (req: Request, res: Response) => {
const { activityId } = req.params; const { activityId } = req.params;
if (!/^\d{1,4}$/.test(activityId)) { if (!/^\d{1,4}$/.test(activityId)) {
return res.status(400).json({ error: 'Invalid Activity ID format.' }); return res.status(400).json({ error: 'Invalid Activity ID format.' });
}
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' });
}
try {
let cachedActivity = await getActivityData(activityId);
const isValidCacheEntry = cachedActivity &&
!cachedActivity.error &&
Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0;
if (isValidCacheEntry) {
logger.info(`Cache HIT for activity ID: ${activityId}`);
cachedActivity.cache = "HIT";
return res.json(cachedActivity);
} }
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.'); logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`);
return res.status(500).json({ error: 'Server configuration error.' }); const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId);
}
liveActivity.cache = "MISS";
try { if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) {
let cachedActivity = await getActivityData(activityId); return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
const isValidCacheEntry = cachedActivity &&
!cachedActivity.error &&
Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0;
if (isValidCacheEntry) {
logger.info(`Cache HIT for activity ID: ${activityId}`);
cachedActivity.cache = "HIT";
return res.json(cachedActivity);
}
logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`);
const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId);
liveActivity.cache = "MISS";
if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) {
return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
}
res.status(status).json(liveActivity);
} catch (error) {
logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
} }
res.status(status).json(liveActivity);
} catch (error) {
logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
}
}); });
// Staff endpoint // Staff endpoint
app.get('/v1/staffs', async (_req: Request, res: Response) => { app.get('/v1/staffs', async (_req: Request, res: Response) => {
if (!USERNAME || !PASSWORD) { if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.'); logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' }); return res.status(500).json({ error: 'Server configuration error.' });
}
try {
let cachedStaffs = await getStaffData();
if (cachedStaffs && cachedStaffs.lastCheck) {
logger.info('Cache HIT for staffs.');
cachedStaffs.cache = "HIT";
return res.json(cachedStaffs);
} }
try { logger.info('Cache MISS for staffs. Fetching from source.');
let cachedStaffs = await getStaffData(); const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID as string, USERNAME, PASSWORD);
if (cachedStaffs && cachedStaffs.lastCheck) { if (activityJson) {
logger.info('Cache HIT for staffs.'); const staffMap = await structStaffData(activityJson);
cachedStaffs.cache = "HIT"; let staffObject: StaffData = Object.fromEntries(staffMap);
return res.json(cachedStaffs); staffObject.lastCheck = new Date().toISOString();
} staffObject.cache = "MISS";
await setStaffData(staffObject);
logger.info('Cache MISS for staffs. Fetching from source.'); res.json(staffObject);
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID as string, USERNAME, PASSWORD); } else {
if (activityJson) { logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
const staffMap = await structStaffData(activityJson); res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" });
let staffObject: StaffData = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
staffObject.cache = "MISS";
await setStaffData(staffObject);
res.json(staffObject);
} else {
logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" });
}
} catch (error) {
logger.error('Error in /v1/staffs endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" });
} }
} catch (error) {
logger.error('Error in /v1/staffs endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" });
}
}); });
// Function to perform background initialization and periodic tasks // Function to perform background initialization and periodic tasks
async function performBackgroundTasks(): Promise<void> { async function performBackgroundTasks(): Promise<void> {
logger.info('Starting background initialization tasks...'); logger.info('Starting background initialization tasks...');
try { try {
await initializeClubCache(); await initializeClubCache();
await initializeOrUpdateStaffCache(true); await initializeOrUpdateStaffCache(true);
await cleanupOrphanedS3Images(); await cleanupOrphanedS3Images();
logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`); logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000); setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000);
logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`); logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000); setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000);
logger.info('Background initialization and periodic task setup complete.'); logger.info('Background initialization and periodic task setup complete.');
} catch (error) { } catch (error) {
logger.error('Error during background initialization tasks:', error); logger.error('Error during background initialization tasks:', error);
} }
} }
// --- Start Server and Background Tasks --- // Start Server and Background Tasks
async function startServer(): Promise<void> { async function startServer(): Promise<void> {
const redis = getRedisClient(); const redis = getRedisClient();
if (!redis) { if (!redis) {
logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.'); logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.');
process.exit(1); process.exit(1);
} }
try { try {
// Test Redis connection with a simple command // Test Redis connection with a simple command
await redis.set('connection-test', 'ok'); await redis.set('connection-test', 'ok');
await redis.del('connection-test'); await redis.del('connection-test');
logger.info('Redis connection confirmed.'); logger.info('Redis connection confirmed.');
app.listen(PORT, () => { app.listen(PORT, () => {
logger.info(`Server is running on http://localhost:${PORT}`); logger.info(`Server is running on http://localhost:${PORT}`);
logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`); logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`);
if (!USERNAME || !PASSWORD) { if (!USERNAME || !PASSWORD) {
logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.'); logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.');
} }
}); });
performBackgroundTasks().catch(error => { performBackgroundTasks().catch(error => {
logger.error('Unhandled error in performBackgroundTasks:', error); logger.error('Unhandled error in performBackgroundTasks:', error);
}); });
} catch (err) { } catch (err) {
logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err); logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err);
process.exit(1); process.exit(1);
} }
} }
// Bun's process event handlers // Bun's process event handlers
process.on('SIGINT', async () => { process.on('SIGINT', async () => {
logger.info('Server shutting down (SIGINT)...'); logger.info('Server shutting down (SIGINT)...');
await closeRedisConnection(); await closeRedisConnection();
process.exit(0); process.exit(0);
}); });
process.on('SIGTERM', async () => { process.on('SIGTERM', async () => {
logger.info('Server shutting down (SIGTERM)...'); logger.info('Server shutting down (SIGTERM)...');
await closeRedisConnection(); await closeRedisConnection();
process.exit(0); process.exit(0);
}); });
// Start the server if not in test mode // Start the server if not in test mode
if (process.env.NODE_ENV !== 'test') { if (process.env.NODE_ENV !== 'test') {
startServer(); startServer();
} }
export { app }; export { app };

4
models/activity.ts
View File

@@ -1,10 +1,10 @@
// src/models/activity.ts // src/models/activity.ts
export interface ActivityData { export interface ActivityData {
// Include all common properties // Include all common properties
id?: string | null | undefined; id?: string | null;
name?: string | null; name?: string | null;
description?: string | null; description?: string | null;
photo?: string | null | undefined; photo?: string | null;
academicYear?: string | null; academicYear?: string | null;
category?: string | null; category?: string | null;
isPreSignup?: boolean | null; isPreSignup?: boolean | null;

1
package.json
View File

@@ -14,6 +14,7 @@
"dependencies": { "dependencies": {
"axios": "^1.9.0", "axios": "^1.9.0",
"cors": "^2.8.5", "cors": "^2.8.5",
"crypto": "^1.0.1",
"dotenv": "^16.5.0", "dotenv": "^16.5.0",
"express": "^5.1.0", "express": "^5.1.0",
"p-limit": "^6.2.0", "p-limit": "^6.2.0",

367
services/cache-manager.ts
View File

@@ -5,12 +5,12 @@ import { fetchActivityData } from '../engage-api/get-activity';
import { structActivityData } from '../engage-api/struct-activity'; import { structActivityData } from '../engage-api/struct-activity';
import { structStaffData } from '../engage-api/struct-staff'; import { structStaffData } from '../engage-api/struct-staff';
import { import {
getActivityData, getActivityData,
setActivityData, setActivityData,
getStaffData, getStaffData,
setStaffData, setStaffData,
getAllActivityKeys, getAllActivityKeys,
ACTIVITY_KEY_PREFIX ACTIVITY_KEY_PREFIX
} from './redis-service'; } from './redis-service';
import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service'; import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service';
import { extractBase64Image } from '../utils/image-processor'; import { extractBase64Image } from '../utils/image-processor';
@@ -40,114 +40,113 @@ const limit = pLimit(CONCURRENT_API_CALLS);
* @returns The processed activity data * @returns The processed activity data
*/ */
async function processAndCacheActivity(activityId: string): Promise<ActivityData> { async function processAndCacheActivity(activityId: string): Promise<ActivityData> {
logger.debug(`Processing activity ID: ${activityId}`); logger.debug(`Processing activity ID: ${activityId}`);
try { try {
if (!USERNAME || !PASSWORD) { if (!USERNAME || !PASSWORD) {
throw new Error('API username or password not configured'); throw new Error('API username or password not configured');
}
const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
let structuredActivity: ActivityData;
if (!activityJson) {
logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`);
structuredActivity = {
lastCheck: new Date().toISOString(),
source: 'api-fetch-empty'
};
} else {
structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo &&
typeof structuredActivity.photo === 'string' &&
structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo);
if (imageInfo) {
const s3Url = await uploadImageFromBase64(
imageInfo.base64Content,
imageInfo.format,
activityId
);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
}
}
}
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
return structuredActivity;
} catch (error) {
logger.error(`Error processing activity ID ${activityId}:`, error);
const errorData: ActivityData = {
lastCheck: new Date().toISOString(),
error: "Failed to fetch or process"
};
await setActivityData(activityId, errorData);
return errorData;
} }
const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
let structuredActivity: ActivityData;
if (!activityJson) {
logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`);
structuredActivity = {
lastCheck: new Date().toISOString(),
source: 'api-fetch-empty'
};
} else {
structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo &&
typeof structuredActivity.photo === 'string' &&
structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo);
if (imageInfo) {
const s3Url = await uploadImageFromBase64(
imageInfo.base64Content,
imageInfo.format,
activityId
);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
}
}
}
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
return structuredActivity;
} catch (error) {
logger.error(`Error processing activity ID ${activityId}:`, error);
const errorData: ActivityData = {
lastCheck: new Date().toISOString(),
error: "Failed to fetch or process"
};
await setActivityData(activityId, errorData);
return errorData;
}
} }
/** /**
* Initialize the club cache by scanning through all activity IDs * Initialize the club cache by scanning through all activity IDs
*/ */
export async function initializeClubCache(): Promise<void> { export async function initializeClubCache(): Promise<void> {
logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`); logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`);
const promises: Promise<void>[] = []; const promises: Promise<void>[] = [];
for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) { for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) {
const activityId = String(i); const activityId = String(i);
promises.push(limit(async () => { promises.push(limit(async () => {
const cachedData = await getActivityData(activityId); const cachedData = await getActivityData(activityId);
if (!cachedData || if (!cachedData ||
Object.keys(cachedData).length === 0 || Object.keys(cachedData).length === 0 ||
!cachedData.lastCheck || !cachedData.lastCheck ||
cachedData.error) { cachedData.error) {
logger.debug(`Initializing cache for activity ID: ${activityId}`); logger.debug(`Initializing cache for activity ID: ${activityId}`);
await processAndCacheActivity(activityId); await processAndCacheActivity(activityId);
} }
})); }));
} }
await Promise.all(promises); await Promise.all(promises);
logger.info('Initial club cache population finished.'); logger.info('Initial club cache population finished.');
} }
/** /**
* Update stale clubs in the cache * Update stale clubs in the cache
*/ */
export async function updateStaleClubs(): Promise<void> { export async function updateStaleClubs(): Promise<void> {
logger.info('Starting stale club check...'); logger.info('Starting stale club check...');
const now = Date.now(); const now = Date.now();
const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000; const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000;
const promises: Promise<void>[] = []; const promises: Promise<void>[] = [];
const activityKeys = await getAllActivityKeys(); const activityKeys = await getAllActivityKeys();
for (const key of activityKeys) { for (const key of activityKeys) {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
promises.push(limit(async () => { promises.push(limit(async () => {
const cachedData = await getActivityData(activityId); const cachedData = await getActivityData(activityId);
if (cachedData && cachedData.lastCheck) { if (cachedData && cachedData.lastCheck) {
const lastCheckTime = new Date(cachedData.lastCheck).getTime(); const lastCheckTime = new Date(cachedData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) { if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) {
logger.info(`Activity ${activityId} is stale or had error. Updating...`); logger.info(`Activity ${activityId} is stale or had error. Updating...`);
await processAndCacheActivity(activityId); await processAndCacheActivity(activityId);
} }
} else if (!cachedData || Object.keys(cachedData).length === 0) { } else if (!cachedData || Object.keys(cachedData).length === 0) {
logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`); logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`);
await processAndCacheActivity(activityId); await processAndCacheActivity(activityId);
} }
})); }));
} }
await cleanupOrphanedS3Images(); await cleanupOrphanedS3Images();
await Promise.all(promises); await Promise.all(promises);
logger.info('Stale club check finished.'); logger.info('Stale club check finished.');
} }
/** /**
@@ -155,99 +154,99 @@ export async function updateStaleClubs(): Promise<void> {
* @param forceUpdate - Force an update regardless of staleness * @param forceUpdate - Force an update regardless of staleness
*/ */
export async function initializeOrUpdateStaffCache(forceUpdate: boolean = false): Promise<void> { export async function initializeOrUpdateStaffCache(forceUpdate: boolean = false): Promise<void> {
logger.info('Starting staff cache check/update...'); logger.info('Starting staff cache check/update...');
try { try {
const cachedStaffData = await getStaffData(); const cachedStaffData = await getStaffData();
const now = Date.now(); const now = Date.now();
const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000; const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000;
let needsUpdate = forceUpdate; let needsUpdate = forceUpdate;
if (!cachedStaffData || !cachedStaffData.lastCheck) { if (!cachedStaffData || !cachedStaffData.lastCheck) {
needsUpdate = true; needsUpdate = true;
} else { } else {
const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime(); const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs) { if ((now - lastCheckTime) > updateIntervalMs) {
needsUpdate = true; needsUpdate = true;
} }
}
if (needsUpdate && USERNAME && PASSWORD && FIXED_STAFF_ACTIVITY_ID) {
logger.info('Staff data needs update. Fetching...');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
const staffObject = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
await setStaffData(staffObject);
logger.info('Staff data updated and cached.');
} else {
logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
if (cachedStaffData && cachedStaffData.lastCheck) {
cachedStaffData.lastCheck = new Date().toISOString();
await setStaffData(cachedStaffData);
}
}
} else {
logger.info('Staff data is up-to-date.');
}
} catch (error) {
logger.error('Error initializing or updating staff cache:', error);
} }
if (needsUpdate && USERNAME && PASSWORD && FIXED_STAFF_ACTIVITY_ID) {
logger.info('Staff data needs update. Fetching...');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
const staffObject = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
await setStaffData(staffObject);
logger.info('Staff data updated and cached.');
} else {
logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
if (cachedStaffData && cachedStaffData.lastCheck) {
cachedStaffData.lastCheck = new Date().toISOString();
await setStaffData(cachedStaffData);
}
}
} else {
logger.info('Staff data is up-to-date.');
}
} catch (error) {
logger.error('Error initializing or updating staff cache:', error);
}
} }
/** /**
* Clean up orphaned S3 images * Clean up orphaned S3 images
*/ */
export async function cleanupOrphanedS3Images(): Promise<void> { export async function cleanupOrphanedS3Images(): Promise<void> {
logger.info('Starting S3 orphan image cleanup...'); logger.info('Starting S3 orphan image cleanup...');
const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : ''; const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : '';
try { try {
const referencedS3Urls = new Set<string>(); const referencedS3Urls = new Set<string>();
const allActivityRedisKeys = await getAllActivityKeys(); const allActivityRedisKeys = await getAllActivityKeys();
const S3_ENDPOINT = process.env.S3_ENDPOINT; const S3_ENDPOINT = process.env.S3_ENDPOINT;
for (const redisKey of allActivityRedisKeys) { for (const redisKey of allActivityRedisKeys) {
const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length); const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length);
const activityData = await getActivityData(activityId); const activityData = await getActivityData(activityId);
if (activityData && if (activityData &&
typeof activityData.photo === 'string' && typeof activityData.photo === 'string' &&
activityData.photo.startsWith('http') && activityData.photo.startsWith('http') &&
S3_ENDPOINT && S3_ENDPOINT &&
activityData.photo.startsWith(S3_ENDPOINT)) { activityData.photo.startsWith(S3_ENDPOINT)) {
referencedS3Urls.add(activityData.photo); referencedS3Urls.add(activityData.photo);
} }
}
logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`);
const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix);
if (!s3ObjectKeys || s3ObjectKeys.length === 0) {
logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`);
return;
}
logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`);
const orphanedObjectKeys: string[] = [];
for (const objectKey of s3ObjectKeys) {
const s3Url = constructS3Url(objectKey);
if (s3Url && !referencedS3Urls.has(s3Url)) {
orphanedObjectKeys.push(objectKey);
}
}
if (orphanedObjectKeys.length > 0) {
logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`);
await deleteS3Objects(orphanedObjectKeys);
} else {
logger.info('No orphaned S3 images found after comparison.');
}
logger.info('S3 orphan image cleanup finished.');
} catch (error) {
logger.error('Error during S3 orphan image cleanup:', error);
} }
logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`);
const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix);
if (!s3ObjectKeys || s3ObjectKeys.length === 0) {
logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`);
return;
}
logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`);
const orphanedObjectKeys: string[] = [];
for (const objectKey of s3ObjectKeys) {
const s3Url = constructS3Url(objectKey);
if (s3Url && !referencedS3Urls.has(s3Url)) {
orphanedObjectKeys.push(objectKey);
}
}
if (orphanedObjectKeys.length > 0) {
logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`);
await deleteS3Objects(orphanedObjectKeys);
} else {
logger.info('No orphaned S3 images found after comparison.');
}
logger.info('S3 orphan image cleanup finished.');
} catch (error) {
logger.error('Error during S3 orphan image cleanup:', error);
}
} }

162
services/redis-service.ts
View File

@@ -13,10 +13,10 @@ const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
let redisClient: RedisClient | null = null; let redisClient: RedisClient | null = null;
try { try {
redisClient = new RedisClient(redisUrl); redisClient = new RedisClient(redisUrl);
logger.info('Redis client initialized. Connection will be established on first command.'); logger.info('Redis client initialized. Connection will be established on first command.');
} catch (error) { } catch (error) {
logger.error('Failed to initialize Redis client:', error); logger.error('Failed to initialize Redis client:', error);
} }
/** /**
@@ -25,17 +25,17 @@ try {
* @returns Parsed JSON object or null if not found/error * @returns Parsed JSON object or null if not found/error
*/ */
export async function getActivityData(activityId: string): Promise<any | null> { export async function getActivityData(activityId: string): Promise<any | null> {
if (!redisClient) { if (!redisClient) {
logger.warn('Redis client not available, skipping getActivityData'); logger.warn('Redis client not available, skipping getActivityData');
return null; return null;
} }
try { try {
const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`); const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`);
return data ? JSON.parse(data) : null; return data ? JSON.parse(data) : null;
} catch (err) { } catch (err) {
logger.error(`Error getting activity ${activityId} from Redis:`, err); logger.error(`Error getting activity ${activityId} from Redis:`, err);
return null; return null;
} }
} }
/** /**
@@ -44,15 +44,15 @@ export async function getActivityData(activityId: string): Promise<any | null> {
* @param data - The activity data object * @param data - The activity data object
*/ */
export async function setActivityData(activityId: string, data: any): Promise<void> { export async function setActivityData(activityId: string, data: any): Promise<void> {
if (!redisClient) { if (!redisClient) {
logger.warn('Redis client not available, skipping setActivityData'); logger.warn('Redis client not available, skipping setActivityData');
return; return;
} }
try { try {
await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data)); await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data));
} catch (err) { } catch (err) {
logger.error(`Error setting activity ${activityId} in Redis:`, err); logger.error(`Error setting activity ${activityId} in Redis:`, err);
} }
} }
/** /**
@@ -60,17 +60,17 @@ export async function setActivityData(activityId: string, data: any): Promise<vo
* @returns Parsed JSON object or null if not found/error * @returns Parsed JSON object or null if not found/error
*/ */
export async function getStaffData(): Promise<any | null> { export async function getStaffData(): Promise<any | null> {
if (!redisClient) { if (!redisClient) {
logger.warn('Redis client not available, skipping getStaffData'); logger.warn('Redis client not available, skipping getStaffData');
return null; return null;
} }
try { try {
const data = await redisClient.get(STAFF_KEY); const data = await redisClient.get(STAFF_KEY);
return data ? JSON.parse(data) : null; return data ? JSON.parse(data) : null;
} catch (err) { } catch (err) {
logger.error('Error getting staff data from Redis:', err); logger.error('Error getting staff data from Redis:', err);
return null; return null;
} }
} }
/** /**
@@ -78,15 +78,15 @@ export async function getStaffData(): Promise<any | null> {
* @param data - The staff data object * @param data - The staff data object
*/ */
export async function setStaffData(data: any): Promise<void> { export async function setStaffData(data: any): Promise<void> {
if (!redisClient) { if (!redisClient) {
logger.warn('Redis client not available, skipping setStaffData'); logger.warn('Redis client not available, skipping setStaffData');
return; return;
} }
try { try {
await redisClient.set(STAFF_KEY, JSON.stringify(data)); await redisClient.set(STAFF_KEY, JSON.stringify(data));
} catch (err) { } catch (err) {
logger.error('Error setting staff data in Redis:', err); logger.error('Error setting staff data in Redis:', err);
} }
} }
/** /**
@@ -95,39 +95,39 @@ export async function setStaffData(data: any): Promise<void> {
* @returns An array of keys * @returns An array of keys
*/ */
export async function getAllActivityKeys(): Promise<string[]> { export async function getAllActivityKeys(): Promise<string[]> {
if (!redisClient) { if (!redisClient) {
logger.warn('Redis client not available, skipping getAllActivityKeys'); logger.warn('Redis client not available, skipping getAllActivityKeys');
return []; return [];
} }
try { try {
// Using raw SCAN command since Bun's RedisClient doesn't have a scan method // Using raw SCAN command since Bun's RedisClient doesn't have a scan method
const keys: string[] = []; const keys: string[] = [];
let cursor = '0'; let cursor = '0';
do { do {
// Use send method to execute raw Redis commands // Use send method to execute raw Redis commands
const result = await redisClient.send('SCAN', [ const result = await redisClient.send('SCAN', [
cursor, cursor,
'MATCH', 'MATCH',
`${ACTIVITY_KEY_PREFIX}*`, `${ACTIVITY_KEY_PREFIX}*`,
'COUNT', 'COUNT',
'100' '100'
]); ]);
cursor = result[0]; cursor = result[0];
const foundKeys = result[1] || []; const foundKeys = result[1] || [];
// Add the found keys to our array // Add the found keys to our array
keys.push(...foundKeys); keys.push(...foundKeys);
} while (cursor !== '0'); } while (cursor !== '0');
logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`); logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`);
return keys; return keys;
} catch (err) { } catch (err) {
logger.error('Error getting all activity keys from Redis using SCAN:', err); logger.error('Error getting all activity keys from Redis using SCAN:', err);
return []; // Return empty array on error return []; // Return empty array on error
} }
} }
/** /**
@@ -135,15 +135,15 @@ export async function getAllActivityKeys(): Promise<string[]> {
* @returns The Redis client or null if not initialized * @returns The Redis client or null if not initialized
*/ */
export function getRedisClient(): RedisClient | null { export function getRedisClient(): RedisClient | null {
return redisClient; return redisClient;
} }
/** /**
* Closes the Redis connection. * Closes the Redis connection.
*/ */
export async function closeRedisConnection(): Promise<void> { export async function closeRedisConnection(): Promise<void> {
if (redisClient) { if (redisClient) {
redisClient.close(); redisClient.close();
logger.info('Redis connection closed.'); logger.info('Redis connection closed.');
} }
} }

302
services/s3-service.ts
View File

@@ -3,6 +3,7 @@ import { S3Client } from "bun";
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import { config } from 'dotenv'; import { config } from 'dotenv';
import sharp from 'sharp'; import sharp from 'sharp';
import crypto from 'crypto';
import { logger } from '../utils/logger'; import { logger } from '../utils/logger';
import { decodeBase64Image } from '../utils/image-processor'; import { decodeBase64Image } from '../utils/image-processor';
@@ -20,75 +21,80 @@ const PUBLIC_URL_FILE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').rep
let s3Client: S3Client | null = null; let s3Client: S3Client | null = null;
if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) { if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {
try { try {
s3Client = new S3Client({ s3Client = new S3Client({
accessKeyId: S3_ACCESS_KEY_ID, accessKeyId: S3_ACCESS_KEY_ID,
secretAccessKey: S3_SECRET_ACCESS_KEY, secretAccessKey: S3_SECRET_ACCESS_KEY,
bucket: BUCKET_NAME, bucket: BUCKET_NAME,
endpoint: S3_ENDPOINT, endpoint: S3_ENDPOINT,
region: S3_REGION region: S3_REGION
}); });
logger.info('S3 client initialized successfully.'); logger.info('S3 client initialized successfully.');
} catch (error) { } catch (error) {
logger.error('Failed to initialize S3 client:', error); logger.error('Failed to initialize S3 client:', error);
} }
} else { } else {
logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.'); logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.');
} }
/** /**
* Uploads an image from a base64 string to S3, converting it to AVIF format. * Uploads an image from a base64 string to S3, converting it to AVIF format.
* Uses MD5 checksum as filename and checks for duplicates before uploading.
* @param base64Data - The base64 content (without the data URI prefix) * @param base64Data - The base64 content (without the data URI prefix)
* @param originalFormat - The image format (e.g., 'png', 'jpeg') * @param originalFormat - The image format (e.g., 'png', 'jpeg')
* @param activityId - The activity ID, used for naming * @param activityId - The activity ID, used for logging purposes
* @returns The public URL of the uploaded image or null on error * @returns The public URL of the uploaded image or null on error
*/ */
export async function uploadImageFromBase64( export async function uploadImageFromBase64(
base64Data: string, base64Data: string,
originalFormat: string, originalFormat: string,
activityId: string activityId: string
): Promise<string | null> { ): Promise<string | null> {
if (!s3Client) { if (!s3Client) {
logger.warn('S3 client not configured. Cannot upload image.'); logger.warn('S3 client not configured. Cannot upload image.');
return null; return null;
} }
if (!base64Data || !originalFormat || !activityId) { if (!base64Data || !originalFormat || !activityId) {
logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId'); logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
return null; return null;
} }
try {
try { // First decode the base64 image
// First decode the base64 image const imageBuffer = decodeBase64Image(base64Data);
const imageBuffer = decodeBase64Image(base64Data); // Convert to AVIF format with quality 80 using Sharp
const avifBuffer = await sharp(imageBuffer)
// Convert to AVIF format with quality 80 using Sharp .avif({
const avifBuffer = await sharp(imageBuffer) quality: 80,
.avif({ // You can add more AVIF options here if needed
quality: 80, // lossless: false,
// You can add more AVIF options here if needed // effort: 4,
// lossless: false, })
// effort: 4, .toBuffer();
}) // Calculate MD5 checksum of the converted AVIF image
.toBuffer(); const md5Hash = crypto.createHash('md5').update(avifBuffer).digest('hex');
const objectKey = `${PUBLIC_URL_FILE_PREFIX}/${md5Hash}.avif`;
// Use .avif extension for the object key // Check if file with this checksum already exists
const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`; const s3File = s3Client.file(objectKey);
const exists = await s3File.exists();
// Using Bun's S3Client file API
const s3File = s3Client.file(objectKey); if (exists) {
const publicUrl = constructS3Url(objectKey);
await s3File.write(avifBuffer, { logger.info(`Image already exists in S3 (MD5: ${md5Hash}), returning existing URL: ${publicUrl}`);
type: 'image/avif', return publicUrl;
acl: 'public-read'
});
const publicUrl = constructS3Url(objectKey);
logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`);
return publicUrl;
} catch (error) {
logger.error(`S3 Upload Error for activity ${activityId}:`, error);
return null;
} }
// File doesn't exist, proceed with upload
await s3File.write(avifBuffer, {
type: 'image/avif',
acl: 'public-read'
});
const publicUrl = constructS3Url(objectKey);
logger.info(`Image uploaded to S3 as AVIF (MD5: ${md5Hash}): ${publicUrl}`);
return publicUrl;
} catch (error) {
logger.error(`S3 Upload Error for activity ${activityId}:`, error);
return null;
}
} }
/** /**
@@ -97,54 +103,49 @@ export async function uploadImageFromBase64(
* @returns A list of object keys * @returns A list of object keys
*/ */
export async function listS3Objects(prefix: string): Promise<string[]> { export async function listS3Objects(prefix: string): Promise<string[]> {
if (!s3Client) { if (!s3Client) {
logger.warn('S3 client not configured. Cannot list objects.'); logger.warn('S3 client not configured. Cannot list objects.');
return []; return [];
} }
logger.debug(`Listing objects from S3 with prefix: "${prefix}"`);
try {
const objectKeys: string[] = [];
let isTruncated = true;
let startAfter: string | undefined;
logger.debug(`Listing objects from S3 with prefix: "${prefix}"`); while (isTruncated) {
// Use Bun's list method with pagination
try { const result = await s3Client.list({
const objectKeys: string[] = []; prefix,
let isTruncated = true; startAfter,
let startAfter: string | undefined; maxKeys: 1000
});
while (isTruncated) { if (result.contents) {
// Use Bun's list method with pagination // Add keys to our array, filtering out "directories"
const result = await s3Client.list({ result.contents.forEach(item => {
prefix, if (item.key && !item.key.endsWith('/')) {
startAfter, objectKeys.push(item.key);
maxKeys: 1000 }
}); });
// Get the last key for pagination
if (result.contents) { if (result.contents?.length > 0) {
// Add keys to our array, filtering out "directories" startAfter = result.contents[result.contents.length - 1]?.key;
result.contents.forEach(item => {
if (item.key && !item.key.endsWith('/')) {
objectKeys.push(item.key);
}
});
// Get the last key for pagination
if (result.contents?.length > 0) {
startAfter = result.contents[result.contents.length - 1]?.key;
}
}
isTruncated = result.isTruncated || false;
// Safety check to prevent infinite loops
if (result.contents?.length === 0) {
break;
}
} }
}
logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`); isTruncated = result.isTruncated || false;
return objectKeys; // Safety check to prevent infinite loops
} catch (error) { if (result.contents?.length === 0) {
logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error); break;
return []; }
} }
logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
return objectKeys;
} catch (error) {
logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error);
return [];
}
} }
/** /**
@@ -153,47 +154,43 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
* @returns True if successful or partially successful, false on major error * @returns True if successful or partially successful, false on major error
*/ */
export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolean> { export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolean> {
if (!s3Client) { if (!s3Client) {
logger.warn('S3 client not configured. Cannot delete objects.'); logger.warn('S3 client not configured. Cannot delete objects.');
return false; return false;
} }
if (!objectKeysArray || objectKeysArray.length === 0) { if (!objectKeysArray || objectKeysArray.length === 0) {
logger.info('No objects to delete from S3.'); logger.info('No objects to delete from S3.');
return true; return true;
} }
try {
try { // With Bun's S3Client, we need to delete objects one by one
// With Bun's S3Client, we need to delete objects one by one // Process in batches of 100 for better performance
// Process in batches of 100 for better performance const BATCH_SIZE = 100;
const BATCH_SIZE = 100; let successCount = 0;
let successCount = 0; let errorCount = 0;
let errorCount = 0;
for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) {
for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) { const batch = objectKeysArray.slice(i, i + BATCH_SIZE);
const batch = objectKeysArray.slice(i, i + BATCH_SIZE); // Process batch in parallel
const results = await Promise.allSettled(
// Process batch in parallel batch.map(key => s3Client!.delete(key))
const results = await Promise.allSettled( );
batch.map(key => s3Client!.delete(key)) // Count successes and failures
); for (const result of results) {
if (result.status === 'fulfilled') {
// Count successes and failures successCount++;
for (const result of results) { } else {
if (result.status === 'fulfilled') { errorCount++;
successCount++; logger.error(`Failed to delete object: ${result.reason}`);
} else {
errorCount++;
logger.error(`Failed to delete object: ${result.reason}`);
}
}
} }
}
logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
return errorCount === 0; // True if all succeeded
} catch (error) {
logger.error('S3 DeleteObjects Error:', error);
return false;
} }
logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
return errorCount === 0; // True if all succeeded
} catch (error) {
logger.error('S3 DeleteObjects Error:', error);
return false;
}
} }
/** /**
@@ -202,16 +199,15 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
* @returns The full public URL * @returns The full public URL
*/ */
export function constructS3Url(objectKey: string): string { export function constructS3Url(objectKey: string): string {
if (!S3_ENDPOINT || !BUCKET_NAME) { if (!S3_ENDPOINT || !BUCKET_NAME) {
return ''; return '';
} }
// Ensure S3_ENDPOINT does not end with a slash
// Ensure S3_ENDPOINT does not end with a slash const s3Base = S3_ENDPOINT.replace(/\/$/, '');
const s3Base = S3_ENDPOINT.replace(/\/$/, ''); // Ensure BUCKET_NAME does not start or end with a slash
// Ensure BUCKET_NAME does not start or end with a slash const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, '');
const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, ''); // Ensure objectKey does not start with a slash
// Ensure objectKey does not start with a slash const key = objectKey.replace(/^\//, '');
const key = objectKey.replace(/^\//, '');
return `${s3Base}/${bucket}/${key}`;
return `${s3Base}/${bucket}/${key}`;
} }

56
utils/image-processor.ts
View File

@@ -24,29 +24,29 @@ interface ImageMarker {
* @returns {ImageInfo|null} An object { base64Content: string, format: string } or null if not found. * @returns {ImageInfo|null} An object { base64Content: string, format: string } or null if not found.
*/ */
export function extractBase64Image(dataUrl: string): ImageInfo | null { export function extractBase64Image(dataUrl: string): ImageInfo | null {
if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) { if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) {
return null;
}
const markers: ImageMarker[] = [
{ prefix: "data:image/png;base64,", format: "png" },
{ prefix: "data:image/jpeg;base64,", format: "jpeg" },
{ prefix: "data:image/jpg;base64,", format: "jpg" },
{ prefix: "data:image/gif;base64,", format: "gif" },
{ prefix: "data:image/svg+xml;base64,", format: "svg" }, // svg+xml -> svg
{ prefix: "data:image/webp;base64,", format: "webp" }
];
for (const marker of markers) {
if (dataUrl.startsWith(marker.prefix)) {
const base64Content = dataUrl.substring(marker.prefix.length);
logger.debug(`Found image of format: ${marker.format}`);
return { base64Content, format: marker.format };
}
}
logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "...");
return null; return null;
}
const markers: ImageMarker[] = [
{ prefix: "data:image/png;base64,", format: "png" },
{ prefix: "data:image/jpeg;base64,", format: "jpeg" },
{ prefix: "data:image/jpg;base64,", format: "jpg" },
{ prefix: "data:image/gif;base64,", format: "gif" },
{ prefix: "data:image/svg+xml;base64,", format: "svg" }, // svg+xml -> svg
{ prefix: "data:image/webp;base64,", format: "webp" }
];
for (const marker of markers) {
if (dataUrl.startsWith(marker.prefix)) {
const base64Content = dataUrl.substring(marker.prefix.length);
logger.debug(`Found image of format: ${marker.format}`);
return { base64Content, format: marker.format };
}
}
logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "...");
return null;
} }
/** /**
@@ -56,8 +56,8 @@ export function extractBase64Image(dataUrl: string): ImageInfo | null {
* @returns {Uint8Array} The decoded binary data * @returns {Uint8Array} The decoded binary data
*/ */
export function decodeBase64Image(base64String: string): Uint8Array { export function decodeBase64Image(base64String: string): Uint8Array {
// Bun uses Node.js Buffer API and has highly optimized Buffer operations // Bun uses Node.js Buffer API and has highly optimized Buffer operations
return Buffer.from(base64String, 'base64'); return Buffer.from(base64String, 'base64');
} }
/** /**
@@ -67,8 +67,8 @@ export function decodeBase64Image(base64String: string): Uint8Array {
* @returns {Uint8Array|null} The decoded image data or null if invalid * @returns {Uint8Array|null} The decoded image data or null if invalid
*/ */
export function dataUrlToBuffer(dataUrl: string): Uint8Array | null { export function dataUrlToBuffer(dataUrl: string): Uint8Array | null {
const imageInfo = extractBase64Image(dataUrl); const imageInfo = extractBase64Image(dataUrl);
if (!imageInfo) return null; if (!imageInfo) return null;
return decodeBase64Image(imageInfo.base64Content); return decodeBase64Image(imageInfo.base64Content);
} }