JamesFlare/dsas-cca-backend-bun
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve: code structure

Browse Source
This commit is contained in:
JamesFlare1212
2025-05-12 21:45:57 -04:00
parent 2100bd04ca
commit 8598571f72
7 changed files with 847 additions and 874 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
engage-api/get-activity.ts
View File

@@ -1,21 +1,21 @@
// ./engage-api/get-activity.ts
// engage-api/get-activity.ts
import axios from 'axios';
import { readFile, writeFile, unlink } from 'fs/promises';
import { readFile,writeFile,unlink } from 'fs/promises';
import { resolve } from 'path';
import { logger } from '../utils/logger';
// Define interfaces for our data structures
interface ActivityResponse {
d: string;
isError?: boolean;
isError ? : boolean;
[key: string]: any;
}
// --- Custom Error for Authentication ---
// Custom Error for Authentication
class AuthenticationError extends Error {
status: number;
constructor(message: string = "Authentication failed, cookie may be invalid.", status?: number) {
constructor(message: string = "Authentication failed, cookie may be invalid.", status ? : number) {
super(message);
this.name = "AuthenticationError";
this.status = status || 0;
@@ -26,8 +26,8 @@ class AuthenticationError extends Error {
const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt');
let _inMemoryCookie: string | null = null;
// --- Cookie Cache Helper Functions ---
async function loadCachedCookie(): Promise<string | null> {
// Cookie Cache Helper Functions
async function loadCachedCookie(): Promise < string | null > {
if (_inMemoryCookie) {
logger.debug("Using in-memory cached cookie.");
return _inMemoryCookie;
@@ -49,7 +49,7 @@ async function loadCachedCookie(): Promise<string | null> {
return null;
}
async function saveCookieToCache(cookieString: string): Promise<void> {
async function saveCookieToCache(cookieString: string): Promise < void > {
if (!cookieString) {
logger.warn("Attempted to save an empty or null cookie. Aborting save.");
return;
@@ -63,7 +63,7 @@ async function saveCookieToCache(cookieString: string): Promise<void> {
}
}
async function clearCookieCache(): Promise<void> {
async function clearCookieCache(): Promise < void > {
_inMemoryCookie = null;
try {
await unlink(COOKIE_FILE_PATH);
@@ -77,13 +77,13 @@ async function clearCookieCache(): Promise<void> {
}
}
async function testCookieValidity(cookieString: string): Promise<boolean> {
async function testCookieValidity(cookieString: string): Promise < boolean > {
if (!cookieString) return false;
logger.debug("Testing cookie validity...");
const MAX_RETRIES = 3;
let attempt = 0;
while (attempt < MAX_RETRIES) {
try {
attempt++;
@@ -93,11 +93,16 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
'Cookie': cookieString,
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
};
const payload = { "activityID": "3350" };
const payload = {
"activityID": "3350"
};
logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`);
await axios.post(url, payload, { headers, timeout: 20000 });
await axios.post(url, payload, {
headers,
timeout: 20000
});
logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
return true;
} catch (error: any) {
@@ -107,7 +112,7 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
} else {
logger.warn(`Network/other error: ${error.message}`);
}
if (attempt >= MAX_RETRIES) {
logger.warn("Max retries reached. Cookie is likely invalid or expired.");
return false;
@@ -117,12 +122,14 @@ async function testCookieValidity(cookieString: string): Promise<boolean> {
return false;
}
// --- Core API Interaction Functions ---
async function getSessionId(): Promise<string | null> {
// Core API Interaction Functions
async function getSessionId(): Promise < string | null > {
const url = 'https://engage.nkcswx.cn/Login.aspx';
try {
const response = await axios.get(url, {
headers: { 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)' }
headers: {
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)'
}
});
const setCookieHeader = response.headers['set-cookie'];
if (setCookieHeader && setCookieHeader.length > 0) {
@@ -141,7 +148,7 @@ async function getSessionId(): Promise<string | null> {
}
}
async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise<string | null> {
async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise < string | null > {
const url = 'https://engage.nkcswx.cn/Login.aspx';
try {
let templateData = await readFile(templateFilePath, 'utf8');
@@ -149,14 +156,14 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
.replace('{{USERNAME}}', userName)
.replace('{{PASSWORD}}', userPwd);
const headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': sessionId,
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
'Referer': 'https://engage.nkcswx.cn/Login.aspx'
};
logger.debug('Getting .ASPXFORMSAUTH');
const response = await axios.post(url, postData, {
headers,
headers,
maxRedirects: 0,
validateStatus: (status) => status >= 200 && status < 400
});
@@ -167,10 +174,10 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
if (aspxAuthCookies.length > 0) {
for (let i = aspxAuthCookies.length - 1; i >= 0; i--) {
const cookieCandidateParts = aspxAuthCookies[i].split(';');
if (cookieCandidateParts.length > 0 && cookieCandidateParts[0] !== undefined) { // Explicit check
if (cookieCandidateParts.length > 0 && cookieCandidateParts[0] !== undefined) { // Explicit check
const firstPart = cookieCandidateParts[0].trim();
if (firstPart.length > '.ASPXFORMSAUTH='.length && firstPart.substring('.ASPXFORMSAUTH='.length).length > 0) {
formsAuthCookieValue = firstPart;
formsAuthCookieValue = firstPart;
break;
}
}
@@ -191,37 +198,39 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t
}
}
async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise<string> {
async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise < string > {
logger.debug('Attempting to get complete cookie string (login process).');
const sessionId = await getSessionId();
if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId.");
const msAuth = await getMSAUTH(sessionId, userName, userPwd, templateFilePath);
if (!msAuth) throw new Error("Login failed: Could not obtain .ASPXFORMSAUTH cookie.");
return `${sessionId}; ${msAuth}`;
}
async function getActivityDetailsRaw(
activityId: string,
cookies: string,
maxRetries: number = 3,
activityId: string,
cookies: string,
maxRetries: number = 3,
timeoutMilliseconds: number = 20000
): Promise<string | null> {
): Promise < string | null > {
const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
const headers = {
'Content-Type': 'application/json; charset=UTF-8',
'Content-Type': 'application/json; charset=UTF-8',
'Cookie': cookies,
'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)',
'X-Requested-With': 'XMLHttpRequest'
};
const payload = { "activityID": String(activityId) };
const payload = {
"activityID": String(activityId)
};
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
const response = await axios.post(url, payload, {
headers,
timeout: timeoutMilliseconds,
headers,
timeout: timeoutMilliseconds,
responseType: 'text'
});
const outerData = JSON.parse(response.data);
@@ -242,12 +251,13 @@ async function getActivityDetailsRaw(
throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
}
logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
if (error.response) {
logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
}
if (attempt === maxRetries - 1) {
logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
throw error;
logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
throw error;
}
await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
}
@@ -265,14 +275,14 @@ async function getActivityDetailsRaw(
* @returns The parsed JSON object of activity details, or null on failure.
*/
export async function fetchActivityData(
activityId: string,
userName: string,
userPwd: string,
templateFileName: string = "login_template.txt",
activityId: string,
userName: string,
userPwd: string,
templateFileName: string = "login_template.txt",
forceLogin: boolean = false
): Promise<any | null> {
): Promise < any | null > {
let currentCookie = forceLogin ? null : await loadCachedCookie();
if (forceLogin && currentCookie) {
await clearCookieCache();
currentCookie = null;
@@ -304,7 +314,7 @@ export async function fetchActivityData(
logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data.");
return null;
}
try {
const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie);
if (rawActivityDetailsString) {
@@ -316,13 +326,13 @@ export async function fetchActivityData(
} catch (error) {
if (error instanceof AuthenticationError) {
logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`);
await clearCookieCache();
await clearCookieCache();
try {
logger.info("Attempting re-login due to authentication failure...");
currentCookie = await getCompleteCookies(userName, userPwd, resolve(import.meta.dir, templateFileName));
await saveCookieToCache(currentCookie);
logger.info("Re-login successful. Retrying request for activity details once...");
const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie);
if (rawActivityDetailsStringRetry) {
@@ -343,4 +353,4 @@ export async function fetchActivityData(
}
// Optionally
export { clearCookieCache, testCookieValidity };
//export { clearCookieCache,testCookieValidity };

728
index.ts
View File

@@ -6,51 +6,41 @@ import { fetchActivityData } from './engage-api/get-activity';
import { structActivityData } from './engage-api/struct-activity';
import { structStaffData } from './engage-api/struct-staff';
import {
getActivityData,
setActivityData,
getStaffData,
setStaffData,
getRedisClient,
getAllActivityKeys,
ACTIVITY_KEY_PREFIX,
closeRedisConnection
getActivityData,
setActivityData,
getStaffData,
setStaffData,
getRedisClient,
getAllActivityKeys,
ACTIVITY_KEY_PREFIX,
closeRedisConnection
} from './services/redis-service';
import { uploadImageFromBase64 } from './services/s3-service';
import { extractBase64Image } from './utils/image-processor';
import {
initializeClubCache,
updateStaleClubs,
initializeOrUpdateStaffCache,
cleanupOrphanedS3Images
initializeClubCache,
updateStaleClubs,
initializeOrUpdateStaffCache,
cleanupOrphanedS3Images
} from './services/cache-manager';
import { logger } from './utils/logger';
import type { ActivityData } from './models/activity'
// Define interfaces for our data structures
interface ActivityData {
id?: string;
name?: string;
photo?: string;
lastCheck?: string;
source?: string;
error?: string;
cache?: string;
[key: string]: any;
}
interface StaffData {
lastCheck?: string;
cache?: string;
[key: string]: any;
lastCheck?: string;
cache?: string;
[key: string]: any;
}
interface ImageInfo {
base64Content: string;
format: string;
base64Content: string;
format: string;
}
interface ProcessedActivityResult {
data: ActivityData;
status: number;
data: ActivityData;
status: number;
}
config();
@@ -65,23 +55,23 @@ const STAFF_CHECK_INTERVAL_SECONDS = parseInt(process.env.STAFF_CHECK_INTERVAL_S
// CORS configuration
type CorsOptions = {
origin: string | string[] | ((origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) => void);
origin: string | string[] | ((origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) => void);
};
let corsOptions: CorsOptions;
if (allowedOriginsEnv === '*') {
corsOptions = { origin: '*' };
corsOptions = { origin: '*' };
} else {
const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim());
corsOptions = {
origin: function (origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) {
if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) {
callback(null, true);
} else {
callback(new Error('Not allowed by CORS'));
}
}
};
const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim());
corsOptions = {
origin: function (origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) {
if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) {
callback(null, true);
} else {
callback(new Error('Not allowed by CORS'));
}
}
};
}
const app = express();
@@ -90,404 +80,386 @@ app.use(express.json());
// Helper function to process activity data (fetch, struct, S3, cache) for API calls
async function fetchProcessAndStoreActivity(activityId: string): Promise<ProcessedActivityResult> {
logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`);
const activityJson = await fetchActivityData(activityId, USERNAME as string, PASSWORD as string);
logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`);
const activityJson = await fetchActivityData(activityId, USERNAME as string, PASSWORD as string);
if (!activityJson) {
logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`);
const emptyData: ActivityData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' };
await setActivityData(activityId, emptyData);
return { data: emptyData, status: 404 };
}
if (!activityJson) {
logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`);
const emptyData: ActivityData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' };
await setActivityData(activityId, emptyData);
return { data: emptyData, status: 404 };
}
let structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo &&
typeof structuredActivity.photo === 'string' &&
structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo) as ImageInfo | null;
if (imageInfo) {
const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
}
}
let structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo &&
typeof structuredActivity.photo === 'string' &&
structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo) as ImageInfo | null;
if (imageInfo) {
const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
}
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
return { data: structuredActivity, status: 200 };
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
return { data: structuredActivity, status: 200 };
}
// --- API Endpoints ---
app.get('/', (_req: Request, res: Response) => {
res.send('Welcome to the DSAS CCA API!<br/>\
GET /v1/activity/list<br/>\
GET /v1/activity/list?category=<br/>\
GET /v1/activity/list?academicYear=<br/>\
GET /v1/activity/list?grade=<br/>\
GET /v1/activity/category<br/>\
GET /v1/activity/academicYear<br/>\
GET /v1/activity/:activityId<br/>\
GET /v1/staffs');
res.send('Welcome to the DSAS CCA API!<br/>\
GET /v1/activity/list<br/>\
GET /v1/activity/list?category=<br/>\
GET /v1/activity/list?academicYear=<br/>\
GET /v1/activity/list?grade=<br/>\
GET /v1/activity/category<br/>\
GET /v1/activity/academicYear<br/>\
GET /v1/activity/:activityId<br/>\
GET /v1/staffs');
});
// Activity list endpoint with filtering capabilities
app.get('/v1/activity/list', async (req: Request, res: Response) => {
try {
const category = req.query.category as string | undefined;
const academicYear = req.query.academicYear as string | undefined;
const grade = req.query.grade as string | undefined;
// Validate academicYear format if provided (YYYY/YYYY)
if (academicYear !== undefined) {
const academicYearRegex = /^\d{4}\/\d{4}$/;
if (!academicYearRegex.test(academicYear)) {
return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' });
}
}
// Validate grade if provided
let validGrade: number | null = null;
if (grade !== undefined) {
const parsedGrade = parseInt(grade, 10);
if (!isNaN(parsedGrade) && parsedGrade > 0 && parsedGrade <= 12) {
validGrade = parsedGrade;
} else {
return res.status(400).json({ error: 'Invalid grade parameter. Must be a number between 1 and 12.' });
}
}
logger.info(`Request received for /v1/activity/list with filters: ${JSON.stringify({category, academicYear, grade: validGrade})}`);
const activityKeys = await getAllActivityKeys();
const clubList: Record<string, {name: string, photo: string}> = {};
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for list.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
// First pass: collect all available categories for validation
const availableCategories = new Set<string>();
const availableAcademicYears = new Set<string>();
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (activityData.category) {
availableCategories.add(activityData.category);
}
if (activityData.academicYear) {
availableAcademicYears.add(activityData.academicYear);
}
}
});
// Validate category against available categories
if (category && !availableCategories.has(category)) {
return res.status(400).json({
error: 'Invalid category parameter. Category not found.',
availableCategories: Array.from(availableCategories)
});
}
// Validate academicYear against available years
if (academicYear && !availableAcademicYears.has(academicYear)) {
return res.status(400).json({
error: 'Invalid academicYear parameter. Academic year not found.',
availableAcademicYears: Array.from(availableAcademicYears)
});
}
// Apply filters and collect club data
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.id &&
activityData.name &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
// Check if it matches category filter if provided
if (category && activityData.category !== category) {
return; // Skip this activity
}
// Check if it matches academicYear filter if provided
if (academicYear && activityData.academicYear !== academicYear) {
return; // Skip this activity
}
// Check if it matches grade filter if provided
if (validGrade !== null) {
// Skip if grades are null
if (!activityData.grades ||
activityData.grades.min === null ||
activityData.grades.max === null) {
return; // Skip this activity
}
const minGrade = parseInt(activityData.grades.min, 10);
const maxGrade = parseInt(activityData.grades.max, 10);
// Skip if grade is out of range or if parsing fails
if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) {
return; // Skip this activity
}
}
// Add to result object with name and photo
clubList[activityData.id] = {
name: activityData.name,
photo: activityData.photo || ""
};
}
});
logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`);
res.json(clubList);
} catch (error) {
logger.error('Error in /v1/activity/list endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating activity list.' });
try {
const category = req.query.category as string | undefined;
const academicYear = req.query.academicYear as string | undefined;
const grade = req.query.grade as string | undefined;
// Validate academicYear format if provided (YYYY/YYYY)
if (academicYear !== undefined) {
const academicYearRegex = /^\d{4}\/\d{4}$/;
if (!academicYearRegex.test(academicYear)) {
return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' });
}
}
// Validate grade if provided
let validGrade: number | null = null;
if (grade !== undefined) {
const parsedGrade = parseInt(grade, 10);
if (!isNaN(parsedGrade) && parsedGrade > 0 && parsedGrade <= 12) {
validGrade = parsedGrade;
} else {
return res.status(400).json({ error: 'Invalid grade parameter. Must be a number between 1 and 12.' });
}
}
logger.info(`Request received for /v1/activity/list with filters: ${JSON.stringify({category, academicYear, grade: validGrade})}`);
const activityKeys = await getAllActivityKeys();
const clubList: Record<string, {name: string, photo: string}> = {};
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for list.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
// First pass: collect all available categories for validation
const availableCategories = new Set<string>();
const availableAcademicYears = new Set<string>();
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (activityData.category) {
availableCategories.add(activityData.category);
}
if (activityData.academicYear) {
availableAcademicYears.add(activityData.academicYear);
}
}
});
// Validate category against available categories
if (category && !availableCategories.has(category)) {
return res.status(400).json({
error: 'Invalid category parameter. Category not found.',
availableCategories: Array.from(availableCategories)
});
}
// Validate academicYear against available years
if (academicYear && !availableAcademicYears.has(academicYear)) {
return res.status(400).json({
error: 'Invalid academicYear parameter. Academic year not found.',
availableAcademicYears: Array.from(availableAcademicYears)
});
}
// Apply filters and collect club data
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.id &&
activityData.name &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
// Check if it matches category filter if provided
if (category && activityData.category !== category) {
return; // Skip this activity
}
// Check if it matches academicYear filter if provided
if (academicYear && activityData.academicYear !== academicYear) {
return; // Skip this activity
}
// Check if it matches grade filter if provided
if (validGrade !== null) {
// Skip if grades are null
if (!activityData.grades ||
activityData.grades.min === null ||
activityData.grades.max === null) {
return; // Skip this activity
}
const minGrade = parseInt(activityData.grades.min, 10);
const maxGrade = parseInt(activityData.grades.max, 10);
// Skip if grade is out of range or if parsing fails
if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) {
return; // Skip this activity
}
}
// Add to result object with name and photo
clubList[activityData.id] = {
name: activityData.name,
photo: activityData.photo || ""
};
}
});
logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`);
res.json(clubList);
} catch (error) {
logger.error('Error in /v1/activity/list endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating activity list.' });
}
});
// Category endpoint
app.get('/v1/activity/category', async (_req: Request, res: Response) => {
try {
logger.info('Request received for /v1/activity/category');
const activityKeys = await getAllActivityKeys();
const categoryMap: Record<string, number> = {};
try {
logger.info('Request received for /v1/activity/category');
const activityKeys = await getAllActivityKeys();
const categoryMap: Record<string, number> = {};
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for categories.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.category &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (categoryMap[activityData.category]) {
categoryMap[activityData.category] = (categoryMap[activityData.category] ?? 0) + 1;
} else {
categoryMap[activityData.category] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`);
res.json(categoryMap);
} catch (error) {
logger.error('Error in /v1/activity/category endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating category list.' });
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for categories.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.category &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (categoryMap[activityData.category]) {
categoryMap[activityData.category] = (categoryMap[activityData.category] ?? 0) + 1;
} else {
categoryMap[activityData.category] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`);
res.json(categoryMap);
} catch (error) {
logger.error('Error in /v1/activity/category endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating category list.' });
}
});
// Academic Year endpoint
app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => {
try {
logger.info('Request received for /v1/activity/academicYear');
const activityKeys = await getAllActivityKeys();
const academicYearMap: Record<string, number> = {};
try {
logger.info('Request received for /v1/activity/academicYear');
const activityKeys = await getAllActivityKeys();
const academicYearMap: Record<string, number> = {};
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for academic years.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.academicYear &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (academicYearMap[activityData.academicYear]) {
academicYearMap[activityData.academicYear] = (academicYearMap[activityData.academicYear] ?? 0) + 1;
} else {
academicYearMap[activityData.academicYear] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`);
res.json(academicYearMap);
} catch (error) {
logger.error('Error in /v1/activity/academicYear endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' });
if (!activityKeys || activityKeys.length === 0) {
logger.info('No activity keys found in Redis for academic years.');
return res.json({});
}
// Fetch all activity data in parallel
const allActivityDataPromises = activityKeys.map(async (key) => {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
return getActivityData(activityId);
});
const allActivities = await Promise.all(allActivityDataPromises);
allActivities.forEach((activityData: ActivityData | null) => {
if (activityData &&
activityData.academicYear &&
!activityData.error &&
activityData.source !== 'api-fetch-empty') {
if (academicYearMap[activityData.academicYear]) {
academicYearMap[activityData.academicYear] = (academicYearMap[activityData.academicYear] ?? 0) + 1;
} else {
academicYearMap[activityData.academicYear] = 1;
}
}
});
logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`);
res.json(academicYearMap);
} catch (error) {
logger.error('Error in /v1/activity/academicYear endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' });
}
});
// Single activity endpoint
app.get('/v1/activity/:activityId', async (req: Request, res: Response) => {
const { activityId } = req.params;
const { activityId } = req.params;
if (!/^\d{1,4}$/.test(activityId)) {
return res.status(400).json({ error: 'Invalid Activity ID format.' });
if (!/^\d{1,4}$/.test(activityId)) {
return res.status(400).json({ error: 'Invalid Activity ID format.' });
}
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' });
}
try {
let cachedActivity = await getActivityData(activityId);
const isValidCacheEntry = cachedActivity &&
!cachedActivity.error &&
Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0;
if (isValidCacheEntry) {
logger.info(`Cache HIT for activity ID: ${activityId}`);
cachedActivity.cache = "HIT";
return res.json(cachedActivity);
}
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' });
}
try {
let cachedActivity = await getActivityData(activityId);
const isValidCacheEntry = cachedActivity &&
!cachedActivity.error &&
Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0;
if (isValidCacheEntry) {
logger.info(`Cache HIT for activity ID: ${activityId}`);
cachedActivity.cache = "HIT";
return res.json(cachedActivity);
}
logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`);
const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId);
liveActivity.cache = "MISS";
if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) {
return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
}
res.status(status).json(liveActivity);
} catch (error) {
logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`);
const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId);
liveActivity.cache = "MISS";
if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) {
return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity });
}
res.status(status).json(liveActivity);
} catch (error) {
logger.error(`Error in /v1/activity/${activityId} endpoint:`, error);
res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" });
}
});
// Staff endpoint
app.get('/v1/staffs', async (_req: Request, res: Response) => {
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' });
if (!USERNAME || !PASSWORD) {
logger.error('API username or password not configured.');
return res.status(500).json({ error: 'Server configuration error.' });
}
try {
let cachedStaffs = await getStaffData();
if (cachedStaffs && cachedStaffs.lastCheck) {
logger.info('Cache HIT for staffs.');
cachedStaffs.cache = "HIT";
return res.json(cachedStaffs);
}
try {
let cachedStaffs = await getStaffData();
if (cachedStaffs && cachedStaffs.lastCheck) {
logger.info('Cache HIT for staffs.');
cachedStaffs.cache = "HIT";
return res.json(cachedStaffs);
}
logger.info('Cache MISS for staffs. Fetching from source.');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID as string, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
let staffObject: StaffData = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
staffObject.cache = "MISS";
await setStaffData(staffObject);
res.json(staffObject);
} else {
logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" });
}
} catch (error) {
logger.error('Error in /v1/staffs endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" });
logger.info('Cache MISS for staffs. Fetching from source.');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID as string, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
let staffObject: StaffData = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
staffObject.cache = "MISS";
await setStaffData(staffObject);
res.json(staffObject);
} else {
logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" });
}
} catch (error) {
logger.error('Error in /v1/staffs endpoint:', error);
res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" });
}
});
// Function to perform background initialization and periodic tasks
async function performBackgroundTasks(): Promise<void> {
logger.info('Starting background initialization tasks...');
try {
await initializeClubCache();
await initializeOrUpdateStaffCache(true);
await cleanupOrphanedS3Images();
logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000);
logger.info('Starting background initialization tasks...');
try {
await initializeClubCache();
await initializeOrUpdateStaffCache(true);
await cleanupOrphanedS3Images();
logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000);
logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000);
logger.info('Background initialization and periodic task setup complete.');
} catch (error) {
logger.error('Error during background initialization tasks:', error);
}
logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`);
setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000);
logger.info('Background initialization and periodic task setup complete.');
} catch (error) {
logger.error('Error during background initialization tasks:', error);
}
}
// --- Start Server and Background Tasks ---
async function startServer(): Promise<void> {
const redis = getRedisClient();
if (!redis) {
logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.');
process.exit(1);
}
const redis = getRedisClient();
if (!redis) {
logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.');
process.exit(1);
}
try {
// Test Redis connection with a simple command
await redis.set('connection-test', 'ok');
await redis.del('connection-test');
logger.info('Redis connection confirmed.');
try {
// Test Redis connection with a simple command
await redis.set('connection-test', 'ok');
await redis.del('connection-test');
logger.info('Redis connection confirmed.');
app.listen(PORT, () => {
logger.info(`Server is running on http://localhost:${PORT}`);
logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`);
if (!USERNAME || !PASSWORD) {
logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.');
}
});
app.listen(PORT, () => {
logger.info(`Server is running on http://localhost:${PORT}`);
logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`);
if (!USERNAME || !PASSWORD) {
logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.');
}
});
performBackgroundTasks().catch(error => {
logger.error('Unhandled error in performBackgroundTasks:', error);
});
performBackgroundTasks().catch(error => {
logger.error('Unhandled error in performBackgroundTasks:', error);
});
} catch (err) {
logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err);
process.exit(1);
}
} catch (err) {
logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err);
process.exit(1);
}
}
// Bun's process event handlers
process.on('SIGINT', async () => {
logger.info('Server shutting down (SIGINT)...');
await closeRedisConnection();
process.exit(0);
logger.info('Server shutting down (SIGINT)...');
await closeRedisConnection();
process.exit(0);
});
process.on('SIGTERM', async () => {
logger.info('Server shutting down (SIGTERM)...');
await closeRedisConnection();
process.exit(0);
logger.info('Server shutting down (SIGTERM)...');
await closeRedisConnection();
process.exit(0);
});
// Start the server if not in test mode
if (process.env.NODE_ENV !== 'test') {
startServer();
startServer();
}
export { app };

4
models/activity.ts
View File

@@ -1,10 +1,10 @@
// src/models/activity.ts
export interface ActivityData {
// Include all common properties
id?: string | null | undefined;
id?: string | null;
name?: string | null;
description?: string | null;
photo?: string | null | undefined;
photo?: string | null;
academicYear?: string | null;
category?: string | null;
isPreSignup?: boolean | null;

367
services/cache-manager.ts
View File

@@ -5,12 +5,12 @@ import { fetchActivityData } from '../engage-api/get-activity';
import { structActivityData } from '../engage-api/struct-activity';
import { structStaffData } from '../engage-api/struct-staff';
import {
getActivityData,
setActivityData,
getStaffData,
setStaffData,
getAllActivityKeys,
ACTIVITY_KEY_PREFIX
getActivityData,
setActivityData,
getStaffData,
setStaffData,
getAllActivityKeys,
ACTIVITY_KEY_PREFIX
} from './redis-service';
import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service';
import { extractBase64Image } from '../utils/image-processor';
@@ -40,114 +40,113 @@ const limit = pLimit(CONCURRENT_API_CALLS);
* @returns The processed activity data
*/
async function processAndCacheActivity(activityId: string): Promise<ActivityData> {
logger.debug(`Processing activity ID: ${activityId}`);
try {
if (!USERNAME || !PASSWORD) {
throw new Error('API username or password not configured');
}
const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
let structuredActivity: ActivityData;
if (!activityJson) {
logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`);
structuredActivity = {
lastCheck: new Date().toISOString(),
source: 'api-fetch-empty'
};
} else {
structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo &&
typeof structuredActivity.photo === 'string' &&
structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo);
if (imageInfo) {
const s3Url = await uploadImageFromBase64(
imageInfo.base64Content,
imageInfo.format,
activityId
);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
}
}
}
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
return structuredActivity;
} catch (error) {
logger.error(`Error processing activity ID ${activityId}:`, error);
const errorData: ActivityData = {
lastCheck: new Date().toISOString(),
error: "Failed to fetch or process"
};
await setActivityData(activityId, errorData);
return errorData;
logger.debug(`Processing activity ID: ${activityId}`);
try {
if (!USERNAME || !PASSWORD) {
throw new Error('API username or password not configured');
}
const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD);
let structuredActivity: ActivityData;
if (!activityJson) {
logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`);
structuredActivity = {
lastCheck: new Date().toISOString(),
source: 'api-fetch-empty'
};
} else {
structuredActivity = await structActivityData(activityJson);
if (structuredActivity && structuredActivity.photo &&
typeof structuredActivity.photo === 'string' &&
structuredActivity.photo.startsWith('data:image')) {
const imageInfo = extractBase64Image(structuredActivity.photo);
if (imageInfo) {
const s3Url = await uploadImageFromBase64(
imageInfo.base64Content,
imageInfo.format,
activityId
);
if (s3Url) {
structuredActivity.photo = s3Url;
} else {
logger.warn(`Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`);
}
}
}
}
structuredActivity.lastCheck = new Date().toISOString();
await setActivityData(activityId, structuredActivity);
return structuredActivity;
} catch (error) {
logger.error(`Error processing activity ID ${activityId}:`, error);
const errorData: ActivityData = {
lastCheck: new Date().toISOString(),
error: "Failed to fetch or process"
};
await setActivityData(activityId, errorData);
return errorData;
}
}
/**
* Initialize the club cache by scanning through all activity IDs
*/
export async function initializeClubCache(): Promise<void> {
logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`);
const promises: Promise<void>[] = [];
for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) {
const activityId = String(i);
promises.push(limit(async () => {
const cachedData = await getActivityData(activityId);
if (!cachedData ||
Object.keys(cachedData).length === 0 ||
!cachedData.lastCheck ||
cachedData.error) {
logger.debug(`Initializing cache for activity ID: ${activityId}`);
await processAndCacheActivity(activityId);
}
}));
}
await Promise.all(promises);
logger.info('Initial club cache population finished.');
logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`);
const promises: Promise<void>[] = [];
for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) {
const activityId = String(i);
promises.push(limit(async () => {
const cachedData = await getActivityData(activityId);
if (!cachedData ||
Object.keys(cachedData).length === 0 ||
!cachedData.lastCheck ||
cachedData.error) {
logger.debug(`Initializing cache for activity ID: ${activityId}`);
await processAndCacheActivity(activityId);
}
}));
}
await Promise.all(promises);
logger.info('Initial club cache population finished.');
}
/**
* Update stale clubs in the cache
*/
export async function updateStaleClubs(): Promise<void> {
logger.info('Starting stale club check...');
const now = Date.now();
const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000;
const promises: Promise<void>[] = [];
const activityKeys = await getAllActivityKeys();
logger.info('Starting stale club check...');
const now = Date.now();
const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000;
const promises: Promise<void>[] = [];
const activityKeys = await getAllActivityKeys();
for (const key of activityKeys) {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
promises.push(limit(async () => {
const cachedData = await getActivityData(activityId);
if (cachedData && cachedData.lastCheck) {
const lastCheckTime = new Date(cachedData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) {
logger.info(`Activity ${activityId} is stale or had error. Updating...`);
await processAndCacheActivity(activityId);
}
} else if (!cachedData || Object.keys(cachedData).length === 0) {
logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`);
await processAndCacheActivity(activityId);
}
}));
}
await cleanupOrphanedS3Images();
await Promise.all(promises);
logger.info('Stale club check finished.');
for (const key of activityKeys) {
const activityId = key.substring(ACTIVITY_KEY_PREFIX.length);
promises.push(limit(async () => {
const cachedData = await getActivityData(activityId);
if (cachedData && cachedData.lastCheck) {
const lastCheckTime = new Date(cachedData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) {
logger.info(`Activity ${activityId} is stale or had error. Updating...`);
await processAndCacheActivity(activityId);
}
} else if (!cachedData || Object.keys(cachedData).length === 0) {
logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`);
await processAndCacheActivity(activityId);
}
}));
}
await cleanupOrphanedS3Images();
await Promise.all(promises);
logger.info('Stale club check finished.');
}
/**
@@ -155,99 +154,99 @@ export async function updateStaleClubs(): Promise<void> {
* @param forceUpdate - Force an update regardless of staleness
*/
export async function initializeOrUpdateStaffCache(forceUpdate: boolean = false): Promise<void> {
logger.info('Starting staff cache check/update...');
try {
const cachedStaffData = await getStaffData();
const now = Date.now();
const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000;
let needsUpdate = forceUpdate;
logger.info('Starting staff cache check/update...');
try {
const cachedStaffData = await getStaffData();
const now = Date.now();
const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000;
let needsUpdate = forceUpdate;
if (!cachedStaffData || !cachedStaffData.lastCheck) {
needsUpdate = true;
} else {
const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs) {
needsUpdate = true;
}
}
if (needsUpdate && USERNAME && PASSWORD && FIXED_STAFF_ACTIVITY_ID) {
logger.info('Staff data needs update. Fetching...');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
const staffObject = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
await setStaffData(staffObject);
logger.info('Staff data updated and cached.');
} else {
logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
if (cachedStaffData && cachedStaffData.lastCheck) {
cachedStaffData.lastCheck = new Date().toISOString();
await setStaffData(cachedStaffData);
}
}
} else {
logger.info('Staff data is up-to-date.');
}
} catch (error) {
logger.error('Error initializing or updating staff cache:', error);
if (!cachedStaffData || !cachedStaffData.lastCheck) {
needsUpdate = true;
} else {
const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime();
if ((now - lastCheckTime) > updateIntervalMs) {
needsUpdate = true;
}
}
if (needsUpdate && USERNAME && PASSWORD && FIXED_STAFF_ACTIVITY_ID) {
logger.info('Staff data needs update. Fetching...');
const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD);
if (activityJson) {
const staffMap = await structStaffData(activityJson);
const staffObject = Object.fromEntries(staffMap);
staffObject.lastCheck = new Date().toISOString();
await setStaffData(staffObject);
logger.info('Staff data updated and cached.');
} else {
logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`);
if (cachedStaffData && cachedStaffData.lastCheck) {
cachedStaffData.lastCheck = new Date().toISOString();
await setStaffData(cachedStaffData);
}
}
} else {
logger.info('Staff data is up-to-date.');
}
} catch (error) {
logger.error('Error initializing or updating staff cache:', error);
}
}
/**
* Clean up orphaned S3 images
*/
export async function cleanupOrphanedS3Images(): Promise<void> {
logger.info('Starting S3 orphan image cleanup...');
const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : '';
logger.info('Starting S3 orphan image cleanup...');
const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : '';
try {
const referencedS3Urls = new Set<string>();
const allActivityRedisKeys = await getAllActivityKeys();
const S3_ENDPOINT = process.env.S3_ENDPOINT;
try {
const referencedS3Urls = new Set<string>();
const allActivityRedisKeys = await getAllActivityKeys();
const S3_ENDPOINT = process.env.S3_ENDPOINT;
for (const redisKey of allActivityRedisKeys) {
const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length);
const activityData = await getActivityData(activityId);
if (activityData &&
typeof activityData.photo === 'string' &&
activityData.photo.startsWith('http') &&
S3_ENDPOINT &&
activityData.photo.startsWith(S3_ENDPOINT)) {
referencedS3Urls.add(activityData.photo);
}
}
logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`);
const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix);
if (!s3ObjectKeys || s3ObjectKeys.length === 0) {
logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`);
return;
}
logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`);
const orphanedObjectKeys: string[] = [];
for (const objectKey of s3ObjectKeys) {
const s3Url = constructS3Url(objectKey);
if (s3Url && !referencedS3Urls.has(s3Url)) {
orphanedObjectKeys.push(objectKey);
}
}
if (orphanedObjectKeys.length > 0) {
logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`);
await deleteS3Objects(orphanedObjectKeys);
} else {
logger.info('No orphaned S3 images found after comparison.');
}
logger.info('S3 orphan image cleanup finished.');
} catch (error) {
logger.error('Error during S3 orphan image cleanup:', error);
for (const redisKey of allActivityRedisKeys) {
const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length);
const activityData = await getActivityData(activityId);
if (activityData &&
typeof activityData.photo === 'string' &&
activityData.photo.startsWith('http') &&
S3_ENDPOINT &&
activityData.photo.startsWith(S3_ENDPOINT)) {
referencedS3Urls.add(activityData.photo);
}
}
logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`);
const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix);
if (!s3ObjectKeys || s3ObjectKeys.length === 0) {
logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`);
return;
}
logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`);
const orphanedObjectKeys: string[] = [];
for (const objectKey of s3ObjectKeys) {
const s3Url = constructS3Url(objectKey);
if (s3Url && !referencedS3Urls.has(s3Url)) {
orphanedObjectKeys.push(objectKey);
}
}
if (orphanedObjectKeys.length > 0) {
logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`);
await deleteS3Objects(orphanedObjectKeys);
} else {
logger.info('No orphaned S3 images found after comparison.');
}
logger.info('S3 orphan image cleanup finished.');
} catch (error) {
logger.error('Error during S3 orphan image cleanup:', error);
}
}

162
services/redis-service.ts
View File

@@ -13,10 +13,10 @@ const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
let redisClient: RedisClient | null = null;
try {
redisClient = new RedisClient(redisUrl);
logger.info('Redis client initialized. Connection will be established on first command.');
redisClient = new RedisClient(redisUrl);
logger.info('Redis client initialized. Connection will be established on first command.');
} catch (error) {
logger.error('Failed to initialize Redis client:', error);
logger.error('Failed to initialize Redis client:', error);
}
/**
@@ -25,17 +25,17 @@ try {
* @returns Parsed JSON object or null if not found/error
*/
export async function getActivityData(activityId: string): Promise<any | null> {
if (!redisClient) {
logger.warn('Redis client not available, skipping getActivityData');
return null;
}
try {
const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`);
return data ? JSON.parse(data) : null;
} catch (err) {
logger.error(`Error getting activity ${activityId} from Redis:`, err);
return null;
}
if (!redisClient) {
logger.warn('Redis client not available, skipping getActivityData');
return null;
}
try {
const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`);
return data ? JSON.parse(data) : null;
} catch (err) {
logger.error(`Error getting activity ${activityId} from Redis:`, err);
return null;
}
}
/**
@@ -44,15 +44,15 @@ export async function getActivityData(activityId: string): Promise<any | null> {
* @param data - The activity data object
*/
export async function setActivityData(activityId: string, data: any): Promise<void> {
if (!redisClient) {
logger.warn('Redis client not available, skipping setActivityData');
return;
}
try {
await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data));
} catch (err) {
logger.error(`Error setting activity ${activityId} in Redis:`, err);
}
if (!redisClient) {
logger.warn('Redis client not available, skipping setActivityData');
return;
}
try {
await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data));
} catch (err) {
logger.error(`Error setting activity ${activityId} in Redis:`, err);
}
}
/**
@@ -60,17 +60,17 @@ export async function setActivityData(activityId: string, data: any): Promise<vo
* @returns Parsed JSON object or null if not found/error
*/
export async function getStaffData(): Promise<any | null> {
if (!redisClient) {
logger.warn('Redis client not available, skipping getStaffData');
return null;
}
try {
const data = await redisClient.get(STAFF_KEY);
return data ? JSON.parse(data) : null;
} catch (err) {
logger.error('Error getting staff data from Redis:', err);
return null;
}
if (!redisClient) {
logger.warn('Redis client not available, skipping getStaffData');
return null;
}
try {
const data = await redisClient.get(STAFF_KEY);
return data ? JSON.parse(data) : null;
} catch (err) {
logger.error('Error getting staff data from Redis:', err);
return null;
}
}
/**
@@ -78,15 +78,15 @@ export async function getStaffData(): Promise<any | null> {
* @param data - The staff data object
*/
export async function setStaffData(data: any): Promise<void> {
if (!redisClient) {
logger.warn('Redis client not available, skipping setStaffData');
return;
}
try {
await redisClient.set(STAFF_KEY, JSON.stringify(data));
} catch (err) {
logger.error('Error setting staff data in Redis:', err);
}
if (!redisClient) {
logger.warn('Redis client not available, skipping setStaffData');
return;
}
try {
await redisClient.set(STAFF_KEY, JSON.stringify(data));
} catch (err) {
logger.error('Error setting staff data in Redis:', err);
}
}
/**
@@ -95,39 +95,39 @@ export async function setStaffData(data: any): Promise<void> {
* @returns An array of keys
*/
export async function getAllActivityKeys(): Promise<string[]> {
if (!redisClient) {
logger.warn('Redis client not available, skipping getAllActivityKeys');
return [];
}
try {
// Using raw SCAN command since Bun's RedisClient doesn't have a scan method
const keys: string[] = [];
let cursor = '0';
do {
// Use send method to execute raw Redis commands
const result = await redisClient.send('SCAN', [
cursor,
'MATCH',
`${ACTIVITY_KEY_PREFIX}*`,
'COUNT',
'100'
]);
cursor = result[0];
const foundKeys = result[1] || [];
// Add the found keys to our array
keys.push(...foundKeys);
} while (cursor !== '0');
logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`);
return keys;
} catch (err) {
logger.error('Error getting all activity keys from Redis using SCAN:', err);
return []; // Return empty array on error
}
if (!redisClient) {
logger.warn('Redis client not available, skipping getAllActivityKeys');
return [];
}
try {
// Using raw SCAN command since Bun's RedisClient doesn't have a scan method
const keys: string[] = [];
let cursor = '0';
do {
// Use send method to execute raw Redis commands
const result = await redisClient.send('SCAN', [
cursor,
'MATCH',
`${ACTIVITY_KEY_PREFIX}*`,
'COUNT',
'100'
]);
cursor = result[0];
const foundKeys = result[1] || [];
// Add the found keys to our array
keys.push(...foundKeys);
} while (cursor !== '0');
logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`);
return keys;
} catch (err) {
logger.error('Error getting all activity keys from Redis using SCAN:', err);
return []; // Return empty array on error
}
}
/**
@@ -135,15 +135,15 @@ export async function getAllActivityKeys(): Promise<string[]> {
* @returns The Redis client or null if not initialized
*/
export function getRedisClient(): RedisClient | null {
return redisClient;
return redisClient;
}
/**
* Closes the Redis connection.
*/
export async function closeRedisConnection(): Promise<void> {
if (redisClient) {
redisClient.close();
logger.info('Redis connection closed.');
}
if (redisClient) {
redisClient.close();
logger.info('Redis connection closed.');
}
}

294
services/s3-service.ts
View File

@@ -20,20 +20,20 @@ const PUBLIC_URL_FILE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').rep
let s3Client: S3Client | null = null;
if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {
try {
s3Client = new S3Client({
accessKeyId: S3_ACCESS_KEY_ID,
secretAccessKey: S3_SECRET_ACCESS_KEY,
bucket: BUCKET_NAME,
endpoint: S3_ENDPOINT,
region: S3_REGION
});
logger.info('S3 client initialized successfully.');
} catch (error) {
logger.error('Failed to initialize S3 client:', error);
}
try {
s3Client = new S3Client({
accessKeyId: S3_ACCESS_KEY_ID,
secretAccessKey: S3_SECRET_ACCESS_KEY,
bucket: BUCKET_NAME,
endpoint: S3_ENDPOINT,
region: S3_REGION
});
logger.info('S3 client initialized successfully.');
} catch (error) {
logger.error('Failed to initialize S3 client:', error);
}
} else {
logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.');
logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.');
}
/**
@@ -44,51 +44,48 @@ if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) {
* @returns The public URL of the uploaded image or null on error
*/
export async function uploadImageFromBase64(
base64Data: string,
originalFormat: string,
activityId: string
base64Data: string,
originalFormat: string,
activityId: string
): Promise<string | null> {
if (!s3Client) {
logger.warn('S3 client not configured. Cannot upload image.');
return null;
}
if (!base64Data || !originalFormat || !activityId) {
logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
return null;
}
if (!s3Client) {
logger.warn('S3 client not configured. Cannot upload image.');
return null;
}
if (!base64Data || !originalFormat || !activityId) {
logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId');
return null;
}
try {
// First decode the base64 image
const imageBuffer = decodeBase64Image(base64Data);
// Convert to AVIF format with quality 80 using Sharp
const avifBuffer = await sharp(imageBuffer)
.avif({
quality: 80,
// You can add more AVIF options here if needed
// lossless: false,
// effort: 4,
})
.toBuffer();
// Use .avif extension for the object key
const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`;
// Using Bun's S3Client file API
const s3File = s3Client.file(objectKey);
await s3File.write(avifBuffer, {
type: 'image/avif',
acl: 'public-read'
});
const publicUrl = constructS3Url(objectKey);
logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`);
return publicUrl;
} catch (error) {
logger.error(`S3 Upload Error for activity ${activityId}:`, error);
return null;
}
try {
// First decode the base64 image
const imageBuffer = decodeBase64Image(base64Data);
// Convert to AVIF format with quality 80 using Sharp
const avifBuffer = await sharp(imageBuffer)
.avif({
quality: 80,
// You can add more AVIF options here if needed
// lossless: false,
// effort: 4,
})
.toBuffer();
// Use .avif extension for the object key
const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`;
// Using Bun's S3Client file API
const s3File = s3Client.file(objectKey);
await s3File.write(avifBuffer, {
type: 'image/avif',
acl: 'public-read'
});
const publicUrl = constructS3Url(objectKey);
logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`);
return publicUrl;
} catch (error) {
logger.error(`S3 Upload Error for activity ${activityId}:`, error);
return null;
}
}
/**
@@ -97,54 +94,52 @@ export async function uploadImageFromBase64(
* @returns A list of object keys
*/
export async function listS3Objects(prefix: string): Promise<string[]> {
if (!s3Client) {
logger.warn('S3 client not configured. Cannot list objects.');
return [];
}
if (!s3Client) {
logger.warn('S3 client not configured. Cannot list objects.');
return [];
}
logger.debug(`Listing objects from S3 with prefix: "${prefix}"`);
try {
const objectKeys: string[] = [];
let isTruncated = true;
let startAfter: string | undefined;
logger.debug(`Listing objects from S3 with prefix: "${prefix}"`);
try {
const objectKeys: string[] = [];
let isTruncated = true;
let startAfter: string | undefined;
while (isTruncated) {
// Use Bun's list method with pagination
const result = await s3Client.list({
prefix,
startAfter,
maxKeys: 1000
});
if (result.contents) {
// Add keys to our array, filtering out "directories"
result.contents.forEach(item => {
if (item.key && !item.key.endsWith('/')) {
objectKeys.push(item.key);
}
});
// Get the last key for pagination
if (result.contents?.length > 0) {
startAfter = result.contents[result.contents.length - 1]?.key;
}
}
isTruncated = result.isTruncated || false;
// Safety check to prevent infinite loops
if (result.contents?.length === 0) {
break;
}
while (isTruncated) {
// Use Bun's list method with pagination
const result = await s3Client.list({
prefix,
startAfter,
maxKeys: 1000
});
if (result.contents) {
// Add keys to our array, filtering out "directories"
result.contents.forEach(item => {
if (item.key && !item.key.endsWith('/')) {
objectKeys.push(item.key);
}
});
// Get the last key for pagination
if (result.contents?.length > 0) {
startAfter = result.contents[result.contents.length - 1]?.key;
}
logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
return objectKeys;
} catch (error) {
logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error);
return [];
}
isTruncated = result.isTruncated || false;
// Safety check to prevent infinite loops
if (result.contents?.length === 0) {
break;
}
}
logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`);
return objectKeys;
} catch (error) {
logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error);
return [];
}
}
/**
@@ -153,47 +148,45 @@ export async function listS3Objects(prefix: string): Promise<string[]> {
* @returns True if successful or partially successful, false on major error
*/
export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolean> {
if (!s3Client) {
logger.warn('S3 client not configured. Cannot delete objects.');
return false;
}
if (!objectKeysArray || objectKeysArray.length === 0) {
logger.info('No objects to delete from S3.');
return true;
}
if (!s3Client) {
logger.warn('S3 client not configured. Cannot delete objects.');
return false;
}
if (!objectKeysArray || objectKeysArray.length === 0) {
logger.info('No objects to delete from S3.');
return true;
}
try {
// With Bun's S3Client, we need to delete objects one by one
// Process in batches of 100 for better performance
const BATCH_SIZE = 100;
let successCount = 0;
let errorCount = 0;
for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) {
const batch = objectKeysArray.slice(i, i + BATCH_SIZE);
// Process batch in parallel
const results = await Promise.allSettled(
batch.map(key => s3Client!.delete(key))
);
// Count successes and failures
for (const result of results) {
if (result.status === 'fulfilled') {
successCount++;
} else {
errorCount++;
logger.error(`Failed to delete object: ${result.reason}`);
}
}
try {
// With Bun's S3Client, we need to delete objects one by one
// Process in batches of 100 for better performance
const BATCH_SIZE = 100;
let successCount = 0;
let errorCount = 0;
for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) {
const batch = objectKeysArray.slice(i, i + BATCH_SIZE);
// Process batch in parallel
const results = await Promise.allSettled(
batch.map(key => s3Client!.delete(key))
);
// Count successes and failures
for (const result of results) {
if (result.status === 'fulfilled') {
successCount++;
} else {
errorCount++;
logger.error(`Failed to delete object: ${result.reason}`);
}
logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
return errorCount === 0; // True if all succeeded
} catch (error) {
logger.error('S3 DeleteObjects Error:', error);
return false;
}
}
logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`);
return errorCount === 0; // True if all succeeded
} catch (error) {
logger.error('S3 DeleteObjects Error:', error);
return false;
}
}
/**
@@ -202,16 +195,15 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise<boolea
* @returns The full public URL
*/
export function constructS3Url(objectKey: string): string {
if (!S3_ENDPOINT || !BUCKET_NAME) {
return '';
}
// Ensure S3_ENDPOINT does not end with a slash
const s3Base = S3_ENDPOINT.replace(/\/$/, '');
// Ensure BUCKET_NAME does not start or end with a slash
const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, '');
// Ensure objectKey does not start with a slash
const key = objectKey.replace(/^\//, '');
return `${s3Base}/${bucket}/${key}`;
if (!S3_ENDPOINT || !BUCKET_NAME) {
return '';
}
// Ensure S3_ENDPOINT does not end with a slash
const s3Base = S3_ENDPOINT.replace(/\/$/, '');
// Ensure BUCKET_NAME does not start or end with a slash
const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, '');
// Ensure objectKey does not start with a slash
const key = objectKey.replace(/^\//, '');
return `${s3Base}/${bucket}/${key}`;
}

56
utils/image-processor.ts
View File

@@ -24,29 +24,29 @@ interface ImageMarker {
* @returns {ImageInfo|null} An object { base64Content: string, format: string } or null if not found.
*/
export function extractBase64Image(dataUrl: string): ImageInfo | null {
if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) {
return null;
}
const markers: ImageMarker[] = [
{ prefix: "data:image/png;base64,", format: "png" },
{ prefix: "data:image/jpeg;base64,", format: "jpeg" },
{ prefix: "data:image/jpg;base64,", format: "jpg" },
{ prefix: "data:image/gif;base64,", format: "gif" },
{ prefix: "data:image/svg+xml;base64,", format: "svg" }, // svg+xml -> svg
{ prefix: "data:image/webp;base64,", format: "webp" }
];
for (const marker of markers) {
if (dataUrl.startsWith(marker.prefix)) {
const base64Content = dataUrl.substring(marker.prefix.length);
logger.debug(`Found image of format: ${marker.format}`);
return { base64Content, format: marker.format };
}
}
logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "...");
if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) {
return null;
}
const markers: ImageMarker[] = [
{ prefix: "data:image/png;base64,", format: "png" },
{ prefix: "data:image/jpeg;base64,", format: "jpeg" },
{ prefix: "data:image/jpg;base64,", format: "jpg" },
{ prefix: "data:image/gif;base64,", format: "gif" },
{ prefix: "data:image/svg+xml;base64,", format: "svg" }, // svg+xml -> svg
{ prefix: "data:image/webp;base64,", format: "webp" }
];
for (const marker of markers) {
if (dataUrl.startsWith(marker.prefix)) {
const base64Content = dataUrl.substring(marker.prefix.length);
logger.debug(`Found image of format: ${marker.format}`);
return { base64Content, format: marker.format };
}
}
logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "...");
return null;
}
/**
@@ -56,8 +56,8 @@ export function extractBase64Image(dataUrl: string): ImageInfo | null {
* @returns {Uint8Array} The decoded binary data
*/
export function decodeBase64Image(base64String: string): Uint8Array {
// Bun uses Node.js Buffer API and has highly optimized Buffer operations
return Buffer.from(base64String, 'base64');
// Bun uses Node.js Buffer API and has highly optimized Buffer operations
return Buffer.from(base64String, 'base64');
}
/**
@@ -67,8 +67,8 @@ export function decodeBase64Image(base64String: string): Uint8Array {
* @returns {Uint8Array|null} The decoded image data or null if invalid
*/
export function dataUrlToBuffer(dataUrl: string): Uint8Array | null {
const imageInfo = extractBase64Image(dataUrl);
if (!imageInfo) return null;
return decodeBase64Image(imageInfo.base64Content);
const imageInfo = extractBase64Image(dataUrl);
if (!imageInfo) return null;
return decodeBase64Image(imageInfo.base64Content);
}