diff --git a/engage-api/get-activity.ts b/engage-api/get-activity.ts index acf69b2..b78bd6a 100644 --- a/engage-api/get-activity.ts +++ b/engage-api/get-activity.ts @@ -1,21 +1,21 @@ -// ./engage-api/get-activity.ts +// engage-api/get-activity.ts import axios from 'axios'; -import { readFile, writeFile, unlink } from 'fs/promises'; +import { readFile,writeFile,unlink } from 'fs/promises'; import { resolve } from 'path'; import { logger } from '../utils/logger'; // Define interfaces for our data structures interface ActivityResponse { d: string; - isError?: boolean; + isError ? : boolean; [key: string]: any; } -// --- Custom Error for Authentication --- +// Custom Error for Authentication class AuthenticationError extends Error { status: number; - - constructor(message: string = "Authentication failed, cookie may be invalid.", status?: number) { + + constructor(message: string = "Authentication failed, cookie may be invalid.", status ? : number) { super(message); this.name = "AuthenticationError"; this.status = status || 0; @@ -26,8 +26,8 @@ class AuthenticationError extends Error { const COOKIE_FILE_PATH = resolve(import.meta.dir, 'nkcs-engage.cookie.txt'); let _inMemoryCookie: string | null = null; -// --- Cookie Cache Helper Functions --- -async function loadCachedCookie(): Promise { +// Cookie Cache Helper Functions +async function loadCachedCookie(): Promise < string | null > { if (_inMemoryCookie) { logger.debug("Using in-memory cached cookie."); return _inMemoryCookie; @@ -49,7 +49,7 @@ async function loadCachedCookie(): Promise { return null; } -async function saveCookieToCache(cookieString: string): Promise { +async function saveCookieToCache(cookieString: string): Promise < void > { if (!cookieString) { logger.warn("Attempted to save an empty or null cookie. Aborting save."); return; @@ -63,7 +63,7 @@ async function saveCookieToCache(cookieString: string): Promise { } } -async function clearCookieCache(): Promise { +async function clearCookieCache(): Promise < void > { _inMemoryCookie = null; try { await unlink(COOKIE_FILE_PATH); @@ -77,13 +77,13 @@ async function clearCookieCache(): Promise { } } -async function testCookieValidity(cookieString: string): Promise { +async function testCookieValidity(cookieString: string): Promise < boolean > { if (!cookieString) return false; logger.debug("Testing cookie validity..."); - + const MAX_RETRIES = 3; let attempt = 0; - + while (attempt < MAX_RETRIES) { try { attempt++; @@ -93,11 +93,16 @@ async function testCookieValidity(cookieString: string): Promise { 'Cookie': cookieString, 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)', }; - const payload = { "activityID": "3350" }; - + const payload = { + "activityID": "3350" + }; + logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`); - await axios.post(url, payload, { headers, timeout: 20000 }); - + await axios.post(url, payload, { + headers, + timeout: 20000 + }); + logger.debug("Cookie test successful (API responded 2xx). Cookie is valid."); return true; } catch (error: any) { @@ -107,7 +112,7 @@ async function testCookieValidity(cookieString: string): Promise { } else { logger.warn(`Network/other error: ${error.message}`); } - + if (attempt >= MAX_RETRIES) { logger.warn("Max retries reached. Cookie is likely invalid or expired."); return false; @@ -117,12 +122,14 @@ async function testCookieValidity(cookieString: string): Promise { return false; } -// --- Core API Interaction Functions --- -async function getSessionId(): Promise { +// Core API Interaction Functions +async function getSessionId(): Promise < string | null > { const url = 'https://engage.nkcswx.cn/Login.aspx'; try { const response = await axios.get(url, { - headers: { 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)' } + headers: { + 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)' + } }); const setCookieHeader = response.headers['set-cookie']; if (setCookieHeader && setCookieHeader.length > 0) { @@ -141,7 +148,7 @@ async function getSessionId(): Promise { } } -async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise { +async function getMSAUTH(sessionId: string, userName: string, userPwd: string, templateFilePath: string): Promise < string | null > { const url = 'https://engage.nkcswx.cn/Login.aspx'; try { let templateData = await readFile(templateFilePath, 'utf8'); @@ -149,14 +156,14 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t .replace('{{USERNAME}}', userName) .replace('{{PASSWORD}}', userPwd); const headers = { - 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Type': 'application/x-www-form-urlencoded', 'Cookie': sessionId, 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)', 'Referer': 'https://engage.nkcswx.cn/Login.aspx' }; logger.debug('Getting .ASPXFORMSAUTH'); const response = await axios.post(url, postData, { - headers, + headers, maxRedirects: 0, validateStatus: (status) => status >= 200 && status < 400 }); @@ -167,10 +174,10 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t if (aspxAuthCookies.length > 0) { for (let i = aspxAuthCookies.length - 1; i >= 0; i--) { const cookieCandidateParts = aspxAuthCookies[i].split(';'); - if (cookieCandidateParts.length > 0 && cookieCandidateParts[0] !== undefined) { // Explicit check + if (cookieCandidateParts.length > 0 && cookieCandidateParts[0] !== undefined) { // Explicit check const firstPart = cookieCandidateParts[0].trim(); if (firstPart.length > '.ASPXFORMSAUTH='.length && firstPart.substring('.ASPXFORMSAUTH='.length).length > 0) { - formsAuthCookieValue = firstPart; + formsAuthCookieValue = firstPart; break; } } @@ -191,37 +198,39 @@ async function getMSAUTH(sessionId: string, userName: string, userPwd: string, t } } -async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise { +async function getCompleteCookies(userName: string, userPwd: string, templateFilePath: string): Promise < string > { logger.debug('Attempting to get complete cookie string (login process).'); const sessionId = await getSessionId(); if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId."); - + const msAuth = await getMSAUTH(sessionId, userName, userPwd, templateFilePath); if (!msAuth) throw new Error("Login failed: Could not obtain .ASPXFORMSAUTH cookie."); - + return `${sessionId}; ${msAuth}`; } async function getActivityDetailsRaw( - activityId: string, - cookies: string, - maxRetries: number = 3, + activityId: string, + cookies: string, + maxRetries: number = 3, timeoutMilliseconds: number = 20000 -): Promise { +): Promise < string | null > { const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails'; const headers = { - 'Content-Type': 'application/json; charset=UTF-8', + 'Content-Type': 'application/json; charset=UTF-8', 'Cookie': cookies, 'User-Agent': 'Mozilla/5.0 (Bun DSAS-CCA get-activity Module)', 'X-Requested-With': 'XMLHttpRequest' }; - const payload = { "activityID": String(activityId) }; + const payload = { + "activityID": String(activityId) + }; for (let attempt = 0; attempt < maxRetries; attempt++) { try { const response = await axios.post(url, payload, { - headers, - timeout: timeoutMilliseconds, + headers, + timeout: timeoutMilliseconds, responseType: 'text' }); const outerData = JSON.parse(response.data); @@ -242,12 +251,13 @@ async function getActivityDetailsRaw( throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status); } logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`); + if (error.response) { - logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`); + logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`); } if (attempt === maxRetries - 1) { - logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`); - throw error; + logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`); + throw error; } await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1))); } @@ -265,14 +275,14 @@ async function getActivityDetailsRaw( * @returns The parsed JSON object of activity details, or null on failure. */ export async function fetchActivityData( - activityId: string, - userName: string, - userPwd: string, - templateFileName: string = "login_template.txt", + activityId: string, + userName: string, + userPwd: string, + templateFileName: string = "login_template.txt", forceLogin: boolean = false -): Promise { +): Promise < any | null > { let currentCookie = forceLogin ? null : await loadCachedCookie(); - + if (forceLogin && currentCookie) { await clearCookieCache(); currentCookie = null; @@ -304,7 +314,7 @@ export async function fetchActivityData( logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data."); return null; } - + try { const rawActivityDetailsString = await getActivityDetailsRaw(activityId, currentCookie); if (rawActivityDetailsString) { @@ -316,13 +326,13 @@ export async function fetchActivityData( } catch (error) { if (error instanceof AuthenticationError) { logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`); - await clearCookieCache(); + await clearCookieCache(); try { logger.info("Attempting re-login due to authentication failure..."); currentCookie = await getCompleteCookies(userName, userPwd, resolve(import.meta.dir, templateFileName)); await saveCookieToCache(currentCookie); - + logger.info("Re-login successful. Retrying request for activity details once..."); const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie); if (rawActivityDetailsStringRetry) { @@ -343,4 +353,4 @@ export async function fetchActivityData( } // Optionally -export { clearCookieCache, testCookieValidity }; \ No newline at end of file +//export { clearCookieCache,testCookieValidity }; \ No newline at end of file diff --git a/index.ts b/index.ts index f6c4bfe..bc05eab 100644 --- a/index.ts +++ b/index.ts @@ -6,51 +6,41 @@ import { fetchActivityData } from './engage-api/get-activity'; import { structActivityData } from './engage-api/struct-activity'; import { structStaffData } from './engage-api/struct-staff'; import { - getActivityData, - setActivityData, - getStaffData, - setStaffData, - getRedisClient, - getAllActivityKeys, - ACTIVITY_KEY_PREFIX, - closeRedisConnection + getActivityData, + setActivityData, + getStaffData, + setStaffData, + getRedisClient, + getAllActivityKeys, + ACTIVITY_KEY_PREFIX, + closeRedisConnection } from './services/redis-service'; import { uploadImageFromBase64 } from './services/s3-service'; import { extractBase64Image } from './utils/image-processor'; import { - initializeClubCache, - updateStaleClubs, - initializeOrUpdateStaffCache, - cleanupOrphanedS3Images + initializeClubCache, + updateStaleClubs, + initializeOrUpdateStaffCache, + cleanupOrphanedS3Images } from './services/cache-manager'; import { logger } from './utils/logger'; +import type { ActivityData } from './models/activity' // Define interfaces for our data structures -interface ActivityData { - id?: string; - name?: string; - photo?: string; - lastCheck?: string; - source?: string; - error?: string; - cache?: string; - [key: string]: any; -} - interface StaffData { - lastCheck?: string; - cache?: string; - [key: string]: any; + lastCheck?: string; + cache?: string; + [key: string]: any; } interface ImageInfo { - base64Content: string; - format: string; + base64Content: string; + format: string; } interface ProcessedActivityResult { - data: ActivityData; - status: number; + data: ActivityData; + status: number; } config(); @@ -65,23 +55,23 @@ const STAFF_CHECK_INTERVAL_SECONDS = parseInt(process.env.STAFF_CHECK_INTERVAL_S // CORS configuration type CorsOptions = { - origin: string | string[] | ((origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) => void); + origin: string | string[] | ((origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) => void); }; let corsOptions: CorsOptions; if (allowedOriginsEnv === '*') { - corsOptions = { origin: '*' }; + corsOptions = { origin: '*' }; } else { - const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim()); - corsOptions = { - origin: function (origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) { - if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) { - callback(null, true); - } else { - callback(new Error('Not allowed by CORS')); - } - } - }; + const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim()); + corsOptions = { + origin: function (origin: string | undefined, callback: (err: Error | null, allow?: boolean) => void) { + if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) { + callback(null, true); + } else { + callback(new Error('Not allowed by CORS')); + } + } + }; } const app = express(); @@ -90,404 +80,386 @@ app.use(express.json()); // Helper function to process activity data (fetch, struct, S3, cache) for API calls async function fetchProcessAndStoreActivity(activityId: string): Promise { - logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`); - const activityJson = await fetchActivityData(activityId, USERNAME as string, PASSWORD as string); + logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`); + const activityJson = await fetchActivityData(activityId, USERNAME as string, PASSWORD as string); - if (!activityJson) { - logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`); - const emptyData: ActivityData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' }; - await setActivityData(activityId, emptyData); - return { data: emptyData, status: 404 }; - } + if (!activityJson) { + logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`); + const emptyData: ActivityData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' }; + await setActivityData(activityId, emptyData); + return { data: emptyData, status: 404 }; + } - let structuredActivity = await structActivityData(activityJson); - if (structuredActivity && structuredActivity.photo && - typeof structuredActivity.photo === 'string' && - structuredActivity.photo.startsWith('data:image')) { - - const imageInfo = extractBase64Image(structuredActivity.photo) as ImageInfo | null; - if (imageInfo) { - const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId); - if (s3Url) { - structuredActivity.photo = s3Url; - } else { - logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`); - } - } + let structuredActivity = await structActivityData(activityJson); + if (structuredActivity && structuredActivity.photo && + typeof structuredActivity.photo === 'string' && + structuredActivity.photo.startsWith('data:image')) { + + const imageInfo = extractBase64Image(structuredActivity.photo) as ImageInfo | null; + if (imageInfo) { + const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId); + if (s3Url) { + structuredActivity.photo = s3Url; + } else { + logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`); + } } - structuredActivity.lastCheck = new Date().toISOString(); - await setActivityData(activityId, structuredActivity); - return { data: structuredActivity, status: 200 }; + } + structuredActivity.lastCheck = new Date().toISOString(); + await setActivityData(activityId, structuredActivity); + return { data: structuredActivity, status: 200 }; } // --- API Endpoints --- app.get('/', (_req: Request, res: Response) => { - res.send('Welcome to the DSAS CCA API!
\ - GET /v1/activity/list
\ - GET /v1/activity/list?category=
\ - GET /v1/activity/list?academicYear=
\ - GET /v1/activity/list?grade=
\ - GET /v1/activity/category
\ - GET /v1/activity/academicYear
\ - GET /v1/activity/:activityId
\ - GET /v1/staffs'); + res.send('Welcome to the DSAS CCA API!
\ + GET /v1/activity/list
\ + GET /v1/activity/list?category=
\ + GET /v1/activity/list?academicYear=
\ + GET /v1/activity/list?grade=
\ + GET /v1/activity/category
\ + GET /v1/activity/academicYear
\ + GET /v1/activity/:activityId
\ + GET /v1/staffs'); }); // Activity list endpoint with filtering capabilities app.get('/v1/activity/list', async (req: Request, res: Response) => { - try { - const category = req.query.category as string | undefined; - const academicYear = req.query.academicYear as string | undefined; - const grade = req.query.grade as string | undefined; - - // Validate academicYear format if provided (YYYY/YYYY) - if (academicYear !== undefined) { - const academicYearRegex = /^\d{4}\/\d{4}$/; - if (!academicYearRegex.test(academicYear)) { - return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' }); - } - } - - // Validate grade if provided - let validGrade: number | null = null; - if (grade !== undefined) { - const parsedGrade = parseInt(grade, 10); - if (!isNaN(parsedGrade) && parsedGrade > 0 && parsedGrade <= 12) { - validGrade = parsedGrade; - } else { - return res.status(400).json({ error: 'Invalid grade parameter. Must be a number between 1 and 12.' }); - } - } - - logger.info(`Request received for /v1/activity/list with filters: ${JSON.stringify({category, academicYear, grade: validGrade})}`); - - const activityKeys = await getAllActivityKeys(); - const clubList: Record = {}; - - if (!activityKeys || activityKeys.length === 0) { - logger.info('No activity keys found in Redis for list.'); - return res.json({}); - } - - // Fetch all activity data in parallel - const allActivityDataPromises = activityKeys.map(async (key) => { - const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); - return getActivityData(activityId); - }); - - const allActivities = await Promise.all(allActivityDataPromises); - - // First pass: collect all available categories for validation - const availableCategories = new Set(); - const availableAcademicYears = new Set(); - - allActivities.forEach((activityData: ActivityData | null) => { - if (activityData && - !activityData.error && - activityData.source !== 'api-fetch-empty') { - if (activityData.category) { - availableCategories.add(activityData.category); - } - if (activityData.academicYear) { - availableAcademicYears.add(activityData.academicYear); - } - } - }); - - // Validate category against available categories - if (category && !availableCategories.has(category)) { - return res.status(400).json({ - error: 'Invalid category parameter. Category not found.', - availableCategories: Array.from(availableCategories) - }); - } - - // Validate academicYear against available years - if (academicYear && !availableAcademicYears.has(academicYear)) { - return res.status(400).json({ - error: 'Invalid academicYear parameter. Academic year not found.', - availableAcademicYears: Array.from(availableAcademicYears) - }); - } - - // Apply filters and collect club data - allActivities.forEach((activityData: ActivityData | null) => { - if (activityData && - activityData.id && - activityData.name && - !activityData.error && - activityData.source !== 'api-fetch-empty') { - - // Check if it matches category filter if provided - if (category && activityData.category !== category) { - return; // Skip this activity - } - - // Check if it matches academicYear filter if provided - if (academicYear && activityData.academicYear !== academicYear) { - return; // Skip this activity - } - - // Check if it matches grade filter if provided - if (validGrade !== null) { - // Skip if grades are null - if (!activityData.grades || - activityData.grades.min === null || - activityData.grades.max === null) { - return; // Skip this activity - } - - const minGrade = parseInt(activityData.grades.min, 10); - const maxGrade = parseInt(activityData.grades.max, 10); - - // Skip if grade is out of range or if parsing fails - if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) { - return; // Skip this activity - } - } - - // Add to result object with name and photo - clubList[activityData.id] = { - name: activityData.name, - photo: activityData.photo || "" - }; - } - }); - - logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`); - res.json(clubList); - - } catch (error) { - logger.error('Error in /v1/activity/list endpoint:', error); - res.status(500).json({ error: 'An internal server error occurred while generating activity list.' }); + try { + const category = req.query.category as string | undefined; + const academicYear = req.query.academicYear as string | undefined; + const grade = req.query.grade as string | undefined; + // Validate academicYear format if provided (YYYY/YYYY) + if (academicYear !== undefined) { + const academicYearRegex = /^\d{4}\/\d{4}$/; + if (!academicYearRegex.test(academicYear)) { + return res.status(400).json({ error: 'Invalid academicYear format. Expected format: YYYY/YYYY' }); + } } + // Validate grade if provided + let validGrade: number | null = null; + if (grade !== undefined) { + const parsedGrade = parseInt(grade, 10); + if (!isNaN(parsedGrade) && parsedGrade > 0 && parsedGrade <= 12) { + validGrade = parsedGrade; + } else { + return res.status(400).json({ error: 'Invalid grade parameter. Must be a number between 1 and 12.' }); + } + } + + logger.info(`Request received for /v1/activity/list with filters: ${JSON.stringify({category, academicYear, grade: validGrade})}`); + + const activityKeys = await getAllActivityKeys(); + const clubList: Record = {}; + + if (!activityKeys || activityKeys.length === 0) { + logger.info('No activity keys found in Redis for list.'); + return res.json({}); + } + // Fetch all activity data in parallel + const allActivityDataPromises = activityKeys.map(async (key) => { + const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); + return getActivityData(activityId); + }); + + const allActivities = await Promise.all(allActivityDataPromises); + // First pass: collect all available categories for validation + const availableCategories = new Set(); + const availableAcademicYears = new Set(); + + allActivities.forEach((activityData: ActivityData | null) => { + if (activityData && + !activityData.error && + activityData.source !== 'api-fetch-empty') { + if (activityData.category) { + availableCategories.add(activityData.category); + } + if (activityData.academicYear) { + availableAcademicYears.add(activityData.academicYear); + } + } + }); + // Validate category against available categories + if (category && !availableCategories.has(category)) { + return res.status(400).json({ + error: 'Invalid category parameter. Category not found.', + availableCategories: Array.from(availableCategories) + }); + } + // Validate academicYear against available years + if (academicYear && !availableAcademicYears.has(academicYear)) { + return res.status(400).json({ + error: 'Invalid academicYear parameter. Academic year not found.', + availableAcademicYears: Array.from(availableAcademicYears) + }); + } + // Apply filters and collect club data + allActivities.forEach((activityData: ActivityData | null) => { + if (activityData && + activityData.id && + activityData.name && + !activityData.error && + activityData.source !== 'api-fetch-empty') { + // Check if it matches category filter if provided + if (category && activityData.category !== category) { + return; // Skip this activity + } + // Check if it matches academicYear filter if provided + if (academicYear && activityData.academicYear !== academicYear) { + return; // Skip this activity + } + // Check if it matches grade filter if provided + if (validGrade !== null) { + // Skip if grades are null + if (!activityData.grades || + activityData.grades.min === null || + activityData.grades.max === null) { + return; // Skip this activity + } + + const minGrade = parseInt(activityData.grades.min, 10); + const maxGrade = parseInt(activityData.grades.max, 10); + // Skip if grade is out of range or if parsing fails + if (isNaN(minGrade) || isNaN(maxGrade) || validGrade < minGrade || validGrade > maxGrade) { + return; // Skip this activity + } + } + // Add to result object with name and photo + clubList[activityData.id] = { + name: activityData.name, + photo: activityData.photo || "" + }; + } + }); + logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs after filtering.`); + res.json(clubList); + + } catch (error) { + logger.error('Error in /v1/activity/list endpoint:', error); + res.status(500).json({ error: 'An internal server error occurred while generating activity list.' }); + } }); // Category endpoint app.get('/v1/activity/category', async (_req: Request, res: Response) => { - try { - logger.info('Request received for /v1/activity/category'); - const activityKeys = await getAllActivityKeys(); - const categoryMap: Record = {}; + try { + logger.info('Request received for /v1/activity/category'); + const activityKeys = await getAllActivityKeys(); + const categoryMap: Record = {}; - if (!activityKeys || activityKeys.length === 0) { - logger.info('No activity keys found in Redis for categories.'); - return res.json({}); - } - - // Fetch all activity data in parallel - const allActivityDataPromises = activityKeys.map(async (key) => { - const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); - return getActivityData(activityId); - }); - - const allActivities = await Promise.all(allActivityDataPromises); - - allActivities.forEach((activityData: ActivityData | null) => { - if (activityData && - activityData.category && - !activityData.error && - activityData.source !== 'api-fetch-empty') { - if (categoryMap[activityData.category]) { - categoryMap[activityData.category] = (categoryMap[activityData.category] ?? 0) + 1; - } else { - categoryMap[activityData.category] = 1; - } - } - }); - - logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`); - res.json(categoryMap); - - } catch (error) { - logger.error('Error in /v1/activity/category endpoint:', error); - res.status(500).json({ error: 'An internal server error occurred while generating category list.' }); + if (!activityKeys || activityKeys.length === 0) { + logger.info('No activity keys found in Redis for categories.'); + return res.json({}); } + // Fetch all activity data in parallel + const allActivityDataPromises = activityKeys.map(async (key) => { + const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); + return getActivityData(activityId); + }); + + const allActivities = await Promise.all(allActivityDataPromises); + + allActivities.forEach((activityData: ActivityData | null) => { + if (activityData && + activityData.category && + !activityData.error && + activityData.source !== 'api-fetch-empty') { + if (categoryMap[activityData.category]) { + categoryMap[activityData.category] = (categoryMap[activityData.category] ?? 0) + 1; + } else { + categoryMap[activityData.category] = 1; + } + } + }); + + logger.info(`Returning list of ${Object.keys(categoryMap).length} categories.`); + res.json(categoryMap); + } catch (error) { + logger.error('Error in /v1/activity/category endpoint:', error); + res.status(500).json({ error: 'An internal server error occurred while generating category list.' }); + } }); // Academic Year endpoint app.get('/v1/activity/academicYear', async (_req: Request, res: Response) => { - try { - logger.info('Request received for /v1/activity/academicYear'); - const activityKeys = await getAllActivityKeys(); - const academicYearMap: Record = {}; + try { + logger.info('Request received for /v1/activity/academicYear'); + const activityKeys = await getAllActivityKeys(); + const academicYearMap: Record = {}; - if (!activityKeys || activityKeys.length === 0) { - logger.info('No activity keys found in Redis for academic years.'); - return res.json({}); - } - - // Fetch all activity data in parallel - const allActivityDataPromises = activityKeys.map(async (key) => { - const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); - return getActivityData(activityId); - }); - - const allActivities = await Promise.all(allActivityDataPromises); - - allActivities.forEach((activityData: ActivityData | null) => { - if (activityData && - activityData.academicYear && - !activityData.error && - activityData.source !== 'api-fetch-empty') { - if (academicYearMap[activityData.academicYear]) { - academicYearMap[activityData.academicYear] = (academicYearMap[activityData.academicYear] ?? 0) + 1; - } else { - academicYearMap[activityData.academicYear] = 1; - } - } - }); - - logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`); - res.json(academicYearMap); - - } catch (error) { - logger.error('Error in /v1/activity/academicYear endpoint:', error); - res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' }); + if (!activityKeys || activityKeys.length === 0) { + logger.info('No activity keys found in Redis for academic years.'); + return res.json({}); } + // Fetch all activity data in parallel + const allActivityDataPromises = activityKeys.map(async (key) => { + const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); + return getActivityData(activityId); + }); + + const allActivities = await Promise.all(allActivityDataPromises); + + allActivities.forEach((activityData: ActivityData | null) => { + if (activityData && + activityData.academicYear && + !activityData.error && + activityData.source !== 'api-fetch-empty') { + if (academicYearMap[activityData.academicYear]) { + academicYearMap[activityData.academicYear] = (academicYearMap[activityData.academicYear] ?? 0) + 1; + } else { + academicYearMap[activityData.academicYear] = 1; + } + } + }); + + logger.info(`Returning list of ${Object.keys(academicYearMap).length} academic years.`); + res.json(academicYearMap); + } catch (error) { + logger.error('Error in /v1/activity/academicYear endpoint:', error); + res.status(500).json({ error: 'An internal server error occurred while generating academic year list.' }); + } }); // Single activity endpoint app.get('/v1/activity/:activityId', async (req: Request, res: Response) => { - const { activityId } = req.params; + const { activityId } = req.params; - if (!/^\d{1,4}$/.test(activityId)) { - return res.status(400).json({ error: 'Invalid Activity ID format.' }); + if (!/^\d{1,4}$/.test(activityId)) { + return res.status(400).json({ error: 'Invalid Activity ID format.' }); + } + if (!USERNAME || !PASSWORD) { + logger.error('API username or password not configured.'); + return res.status(500).json({ error: 'Server configuration error.' }); + } + + try { + let cachedActivity = await getActivityData(activityId); + const isValidCacheEntry = cachedActivity && + !cachedActivity.error && + Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0; + + if (isValidCacheEntry) { + logger.info(`Cache HIT for activity ID: ${activityId}`); + cachedActivity.cache = "HIT"; + return res.json(cachedActivity); } - if (!USERNAME || !PASSWORD) { - logger.error('API username or password not configured.'); - return res.status(500).json({ error: 'Server configuration error.' }); - } - - try { - let cachedActivity = await getActivityData(activityId); - const isValidCacheEntry = cachedActivity && - !cachedActivity.error && - Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0; - - if (isValidCacheEntry) { - logger.info(`Cache HIT for activity ID: ${activityId}`); - cachedActivity.cache = "HIT"; - return res.json(cachedActivity); - } - - logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`); - const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId); - - liveActivity.cache = "MISS"; - if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) { - return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity }); - } - res.status(status).json(liveActivity); - - } catch (error) { - logger.error(`Error in /v1/activity/${activityId} endpoint:`, error); - res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" }); + + logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`); + const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId); + + liveActivity.cache = "MISS"; + if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) { + return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity }); } + res.status(status).json(liveActivity); + } catch (error) { + logger.error(`Error in /v1/activity/${activityId} endpoint:`, error); + res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" }); + } }); // Staff endpoint app.get('/v1/staffs', async (_req: Request, res: Response) => { - if (!USERNAME || !PASSWORD) { - logger.error('API username or password not configured.'); - return res.status(500).json({ error: 'Server configuration error.' }); + if (!USERNAME || !PASSWORD) { + logger.error('API username or password not configured.'); + return res.status(500).json({ error: 'Server configuration error.' }); + } + + try { + let cachedStaffs = await getStaffData(); + if (cachedStaffs && cachedStaffs.lastCheck) { + logger.info('Cache HIT for staffs.'); + cachedStaffs.cache = "HIT"; + return res.json(cachedStaffs); } - try { - let cachedStaffs = await getStaffData(); - if (cachedStaffs && cachedStaffs.lastCheck) { - logger.info('Cache HIT for staffs.'); - cachedStaffs.cache = "HIT"; - return res.json(cachedStaffs); - } - - logger.info('Cache MISS for staffs. Fetching from source.'); - const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID as string, USERNAME, PASSWORD); - if (activityJson) { - const staffMap = await structStaffData(activityJson); - let staffObject: StaffData = Object.fromEntries(staffMap); - staffObject.lastCheck = new Date().toISOString(); - staffObject.cache = "MISS"; - await setStaffData(staffObject); - res.json(staffObject); - } else { - logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`); - res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" }); - } - } catch (error) { - logger.error('Error in /v1/staffs endpoint:', error); - res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" }); + logger.info('Cache MISS for staffs. Fetching from source.'); + const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID as string, USERNAME, PASSWORD); + if (activityJson) { + const staffMap = await structStaffData(activityJson); + let staffObject: StaffData = Object.fromEntries(staffMap); + staffObject.lastCheck = new Date().toISOString(); + staffObject.cache = "MISS"; + await setStaffData(staffObject); + res.json(staffObject); + } else { + logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`); + res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" }); } + } catch (error) { + logger.error('Error in /v1/staffs endpoint:', error); + res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" }); + } }); // Function to perform background initialization and periodic tasks async function performBackgroundTasks(): Promise { - logger.info('Starting background initialization tasks...'); - try { - await initializeClubCache(); - await initializeOrUpdateStaffCache(true); - await cleanupOrphanedS3Images(); - - logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`); - setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000); + logger.info('Starting background initialization tasks...'); + try { + await initializeClubCache(); + await initializeOrUpdateStaffCache(true); + await cleanupOrphanedS3Images(); + + logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`); + setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000); - logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`); - setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000); - - logger.info('Background initialization and periodic task setup complete.'); - } catch (error) { - logger.error('Error during background initialization tasks:', error); - } + logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`); + setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000); + + logger.info('Background initialization and periodic task setup complete.'); + } catch (error) { + logger.error('Error during background initialization tasks:', error); + } } // --- Start Server and Background Tasks --- async function startServer(): Promise { - const redis = getRedisClient(); - if (!redis) { - logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.'); - process.exit(1); - } + const redis = getRedisClient(); + if (!redis) { + logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.'); + process.exit(1); + } - try { - // Test Redis connection with a simple command - await redis.set('connection-test', 'ok'); - await redis.del('connection-test'); - logger.info('Redis connection confirmed.'); + try { + // Test Redis connection with a simple command + await redis.set('connection-test', 'ok'); + await redis.del('connection-test'); + logger.info('Redis connection confirmed.'); - app.listen(PORT, () => { - logger.info(`Server is running on http://localhost:${PORT}`); - logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`); - if (!USERNAME || !PASSWORD) { - logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.'); - } - }); + app.listen(PORT, () => { + logger.info(`Server is running on http://localhost:${PORT}`); + logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`); + if (!USERNAME || !PASSWORD) { + logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.'); + } + }); - performBackgroundTasks().catch(error => { - logger.error('Unhandled error in performBackgroundTasks:', error); - }); + performBackgroundTasks().catch(error => { + logger.error('Unhandled error in performBackgroundTasks:', error); + }); - } catch (err) { - logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err); - process.exit(1); - } + } catch (err) { + logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err); + process.exit(1); + } } // Bun's process event handlers process.on('SIGINT', async () => { - logger.info('Server shutting down (SIGINT)...'); - await closeRedisConnection(); - process.exit(0); + logger.info('Server shutting down (SIGINT)...'); + await closeRedisConnection(); + process.exit(0); }); process.on('SIGTERM', async () => { - logger.info('Server shutting down (SIGTERM)...'); - await closeRedisConnection(); - process.exit(0); + logger.info('Server shutting down (SIGTERM)...'); + await closeRedisConnection(); + process.exit(0); }); // Start the server if not in test mode if (process.env.NODE_ENV !== 'test') { - startServer(); + startServer(); } export { app }; \ No newline at end of file diff --git a/models/activity.ts b/models/activity.ts index 35e1b6e..bf09544 100644 --- a/models/activity.ts +++ b/models/activity.ts @@ -1,10 +1,10 @@ // src/models/activity.ts export interface ActivityData { // Include all common properties - id?: string | null | undefined; + id?: string | null; name?: string | null; description?: string | null; - photo?: string | null | undefined; + photo?: string | null; academicYear?: string | null; category?: string | null; isPreSignup?: boolean | null; diff --git a/services/cache-manager.ts b/services/cache-manager.ts index 642fb76..c7df654 100644 --- a/services/cache-manager.ts +++ b/services/cache-manager.ts @@ -5,12 +5,12 @@ import { fetchActivityData } from '../engage-api/get-activity'; import { structActivityData } from '../engage-api/struct-activity'; import { structStaffData } from '../engage-api/struct-staff'; import { - getActivityData, - setActivityData, - getStaffData, - setStaffData, - getAllActivityKeys, - ACTIVITY_KEY_PREFIX + getActivityData, + setActivityData, + getStaffData, + setStaffData, + getAllActivityKeys, + ACTIVITY_KEY_PREFIX } from './redis-service'; import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service'; import { extractBase64Image } from '../utils/image-processor'; @@ -40,114 +40,113 @@ const limit = pLimit(CONCURRENT_API_CALLS); * @returns The processed activity data */ async function processAndCacheActivity(activityId: string): Promise { - logger.debug(`Processing activity ID: ${activityId}`); - try { - if (!USERNAME || !PASSWORD) { - throw new Error('API username or password not configured'); - } - - const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD); - let structuredActivity: ActivityData; - - if (!activityJson) { - logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`); - structuredActivity = { - lastCheck: new Date().toISOString(), - source: 'api-fetch-empty' - }; - } else { - structuredActivity = await structActivityData(activityJson); - if (structuredActivity && structuredActivity.photo && - typeof structuredActivity.photo === 'string' && - structuredActivity.photo.startsWith('data:image')) { - - const imageInfo = extractBase64Image(structuredActivity.photo); - if (imageInfo) { - const s3Url = await uploadImageFromBase64( - imageInfo.base64Content, - imageInfo.format, - activityId - ); - - if (s3Url) { - structuredActivity.photo = s3Url; - } else { - logger.warn(`Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`); - } - } - } - } - - structuredActivity.lastCheck = new Date().toISOString(); - await setActivityData(activityId, structuredActivity); - return structuredActivity; - } catch (error) { - logger.error(`Error processing activity ID ${activityId}:`, error); - const errorData: ActivityData = { - lastCheck: new Date().toISOString(), - error: "Failed to fetch or process" - }; - await setActivityData(activityId, errorData); - return errorData; + logger.debug(`Processing activity ID: ${activityId}`); + try { + if (!USERNAME || !PASSWORD) { + throw new Error('API username or password not configured'); } + + const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD); + let structuredActivity: ActivityData; + + if (!activityJson) { + logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`); + structuredActivity = { + lastCheck: new Date().toISOString(), + source: 'api-fetch-empty' + }; + } else { + structuredActivity = await structActivityData(activityJson); + if (structuredActivity && structuredActivity.photo && + typeof structuredActivity.photo === 'string' && + structuredActivity.photo.startsWith('data:image')) { + + const imageInfo = extractBase64Image(structuredActivity.photo); + if (imageInfo) { + const s3Url = await uploadImageFromBase64( + imageInfo.base64Content, + imageInfo.format, + activityId + ); + + if (s3Url) { + structuredActivity.photo = s3Url; + } else { + logger.warn(`Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`); + } + } + } + } + structuredActivity.lastCheck = new Date().toISOString(); + await setActivityData(activityId, structuredActivity); + return structuredActivity; + } catch (error) { + logger.error(`Error processing activity ID ${activityId}:`, error); + const errorData: ActivityData = { + lastCheck: new Date().toISOString(), + error: "Failed to fetch or process" + }; + await setActivityData(activityId, errorData); + return errorData; + } } /** * Initialize the club cache by scanning through all activity IDs */ export async function initializeClubCache(): Promise { - logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`); - const promises: Promise[] = []; - - for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) { - const activityId = String(i); - promises.push(limit(async () => { - const cachedData = await getActivityData(activityId); - if (!cachedData || - Object.keys(cachedData).length === 0 || - !cachedData.lastCheck || - cachedData.error) { - logger.debug(`Initializing cache for activity ID: ${activityId}`); - await processAndCacheActivity(activityId); - } - })); - } - - await Promise.all(promises); - logger.info('Initial club cache population finished.'); + logger.info(`Starting initial club cache population from ID ${MIN_ACTIVITY_ID_SCAN} to ${MAX_ACTIVITY_ID_SCAN}`); + const promises: Promise[] = []; + + for (let i = MIN_ACTIVITY_ID_SCAN; i <= MAX_ACTIVITY_ID_SCAN; i++) { + const activityId = String(i); + promises.push(limit(async () => { + const cachedData = await getActivityData(activityId); + if (!cachedData || + Object.keys(cachedData).length === 0 || + !cachedData.lastCheck || + cachedData.error) { + logger.debug(`Initializing cache for activity ID: ${activityId}`); + await processAndCacheActivity(activityId); + } + })); + } + + await Promise.all(promises); + logger.info('Initial club cache population finished.'); } /** * Update stale clubs in the cache */ export async function updateStaleClubs(): Promise { - logger.info('Starting stale club check...'); - const now = Date.now(); - const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000; - const promises: Promise[] = []; - const activityKeys = await getAllActivityKeys(); + logger.info('Starting stale club check...'); + const now = Date.now(); + const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000; + const promises: Promise[] = []; + const activityKeys = await getAllActivityKeys(); - for (const key of activityKeys) { - const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); - promises.push(limit(async () => { - const cachedData = await getActivityData(activityId); - - if (cachedData && cachedData.lastCheck) { - const lastCheckTime = new Date(cachedData.lastCheck).getTime(); - if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) { - logger.info(`Activity ${activityId} is stale or had error. Updating...`); - await processAndCacheActivity(activityId); - } - } else if (!cachedData || Object.keys(cachedData).length === 0) { - logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`); - await processAndCacheActivity(activityId); - } - })); - } - - await cleanupOrphanedS3Images(); - await Promise.all(promises); - logger.info('Stale club check finished.'); + for (const key of activityKeys) { + const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); + promises.push(limit(async () => { + const cachedData = await getActivityData(activityId); + + if (cachedData && cachedData.lastCheck) { + const lastCheckTime = new Date(cachedData.lastCheck).getTime(); + if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) { + logger.info(`Activity ${activityId} is stale or had error. Updating...`); + await processAndCacheActivity(activityId); + } + } else if (!cachedData || Object.keys(cachedData).length === 0) { + logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`); + await processAndCacheActivity(activityId); + } + })); + } + + await cleanupOrphanedS3Images(); + await Promise.all(promises); + logger.info('Stale club check finished.'); } /** @@ -155,99 +154,99 @@ export async function updateStaleClubs(): Promise { * @param forceUpdate - Force an update regardless of staleness */ export async function initializeOrUpdateStaffCache(forceUpdate: boolean = false): Promise { - logger.info('Starting staff cache check/update...'); - try { - const cachedStaffData = await getStaffData(); - const now = Date.now(); - const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000; - let needsUpdate = forceUpdate; + logger.info('Starting staff cache check/update...'); + try { + const cachedStaffData = await getStaffData(); + const now = Date.now(); + const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000; + let needsUpdate = forceUpdate; - if (!cachedStaffData || !cachedStaffData.lastCheck) { - needsUpdate = true; - } else { - const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime(); - if ((now - lastCheckTime) > updateIntervalMs) { - needsUpdate = true; - } - } - - if (needsUpdate && USERNAME && PASSWORD && FIXED_STAFF_ACTIVITY_ID) { - logger.info('Staff data needs update. Fetching...'); - const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD); - - if (activityJson) { - const staffMap = await structStaffData(activityJson); - const staffObject = Object.fromEntries(staffMap); - staffObject.lastCheck = new Date().toISOString(); - await setStaffData(staffObject); - logger.info('Staff data updated and cached.'); - } else { - logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`); - if (cachedStaffData && cachedStaffData.lastCheck) { - cachedStaffData.lastCheck = new Date().toISOString(); - await setStaffData(cachedStaffData); - } - } - } else { - logger.info('Staff data is up-to-date.'); - } - } catch (error) { - logger.error('Error initializing or updating staff cache:', error); + if (!cachedStaffData || !cachedStaffData.lastCheck) { + needsUpdate = true; + } else { + const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime(); + if ((now - lastCheckTime) > updateIntervalMs) { + needsUpdate = true; + } } + + if (needsUpdate && USERNAME && PASSWORD && FIXED_STAFF_ACTIVITY_ID) { + logger.info('Staff data needs update. Fetching...'); + const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD); + + if (activityJson) { + const staffMap = await structStaffData(activityJson); + const staffObject = Object.fromEntries(staffMap); + staffObject.lastCheck = new Date().toISOString(); + await setStaffData(staffObject); + logger.info('Staff data updated and cached.'); + } else { + logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`); + if (cachedStaffData && cachedStaffData.lastCheck) { + cachedStaffData.lastCheck = new Date().toISOString(); + await setStaffData(cachedStaffData); + } + } + } else { + logger.info('Staff data is up-to-date.'); + } + } catch (error) { + logger.error('Error initializing or updating staff cache:', error); + } } /** * Clean up orphaned S3 images */ export async function cleanupOrphanedS3Images(): Promise { - logger.info('Starting S3 orphan image cleanup...'); - const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : ''; + logger.info('Starting S3 orphan image cleanup...'); + const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : ''; - try { - const referencedS3Urls = new Set(); - const allActivityRedisKeys = await getAllActivityKeys(); - const S3_ENDPOINT = process.env.S3_ENDPOINT; + try { + const referencedS3Urls = new Set(); + const allActivityRedisKeys = await getAllActivityKeys(); + const S3_ENDPOINT = process.env.S3_ENDPOINT; - for (const redisKey of allActivityRedisKeys) { - const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length); - const activityData = await getActivityData(activityId); - - if (activityData && - typeof activityData.photo === 'string' && - activityData.photo.startsWith('http') && - S3_ENDPOINT && - activityData.photo.startsWith(S3_ENDPOINT)) { - referencedS3Urls.add(activityData.photo); - } - } - - logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`); - - const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix); - if (!s3ObjectKeys || s3ObjectKeys.length === 0) { - logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`); - return; - } - - logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`); - - const orphanedObjectKeys: string[] = []; - for (const objectKey of s3ObjectKeys) { - const s3Url = constructS3Url(objectKey); - if (s3Url && !referencedS3Urls.has(s3Url)) { - orphanedObjectKeys.push(objectKey); - } - } - - if (orphanedObjectKeys.length > 0) { - logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`); - await deleteS3Objects(orphanedObjectKeys); - } else { - logger.info('No orphaned S3 images found after comparison.'); - } - - logger.info('S3 orphan image cleanup finished.'); - } catch (error) { - logger.error('Error during S3 orphan image cleanup:', error); + for (const redisKey of allActivityRedisKeys) { + const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length); + const activityData = await getActivityData(activityId); + + if (activityData && + typeof activityData.photo === 'string' && + activityData.photo.startsWith('http') && + S3_ENDPOINT && + activityData.photo.startsWith(S3_ENDPOINT)) { + referencedS3Urls.add(activityData.photo); + } } + + logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`); + + const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix); + if (!s3ObjectKeys || s3ObjectKeys.length === 0) { + logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`); + return; + } + + logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`); + + const orphanedObjectKeys: string[] = []; + for (const objectKey of s3ObjectKeys) { + const s3Url = constructS3Url(objectKey); + if (s3Url && !referencedS3Urls.has(s3Url)) { + orphanedObjectKeys.push(objectKey); + } + } + + if (orphanedObjectKeys.length > 0) { + logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`); + await deleteS3Objects(orphanedObjectKeys); + } else { + logger.info('No orphaned S3 images found after comparison.'); + } + + logger.info('S3 orphan image cleanup finished.'); + } catch (error) { + logger.error('Error during S3 orphan image cleanup:', error); + } } diff --git a/services/redis-service.ts b/services/redis-service.ts index 25c0765..db52e1a 100644 --- a/services/redis-service.ts +++ b/services/redis-service.ts @@ -13,10 +13,10 @@ const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379'; let redisClient: RedisClient | null = null; try { - redisClient = new RedisClient(redisUrl); - logger.info('Redis client initialized. Connection will be established on first command.'); + redisClient = new RedisClient(redisUrl); + logger.info('Redis client initialized. Connection will be established on first command.'); } catch (error) { - logger.error('Failed to initialize Redis client:', error); + logger.error('Failed to initialize Redis client:', error); } /** @@ -25,17 +25,17 @@ try { * @returns Parsed JSON object or null if not found/error */ export async function getActivityData(activityId: string): Promise { - if (!redisClient) { - logger.warn('Redis client not available, skipping getActivityData'); - return null; - } - try { - const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`); - return data ? JSON.parse(data) : null; - } catch (err) { - logger.error(`Error getting activity ${activityId} from Redis:`, err); - return null; - } + if (!redisClient) { + logger.warn('Redis client not available, skipping getActivityData'); + return null; + } + try { + const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`); + return data ? JSON.parse(data) : null; + } catch (err) { + logger.error(`Error getting activity ${activityId} from Redis:`, err); + return null; + } } /** @@ -44,15 +44,15 @@ export async function getActivityData(activityId: string): Promise { * @param data - The activity data object */ export async function setActivityData(activityId: string, data: any): Promise { - if (!redisClient) { - logger.warn('Redis client not available, skipping setActivityData'); - return; - } - try { - await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data)); - } catch (err) { - logger.error(`Error setting activity ${activityId} in Redis:`, err); - } + if (!redisClient) { + logger.warn('Redis client not available, skipping setActivityData'); + return; + } + try { + await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data)); + } catch (err) { + logger.error(`Error setting activity ${activityId} in Redis:`, err); + } } /** @@ -60,17 +60,17 @@ export async function setActivityData(activityId: string, data: any): Promise { - if (!redisClient) { - logger.warn('Redis client not available, skipping getStaffData'); - return null; - } - try { - const data = await redisClient.get(STAFF_KEY); - return data ? JSON.parse(data) : null; - } catch (err) { - logger.error('Error getting staff data from Redis:', err); - return null; - } + if (!redisClient) { + logger.warn('Redis client not available, skipping getStaffData'); + return null; + } + try { + const data = await redisClient.get(STAFF_KEY); + return data ? JSON.parse(data) : null; + } catch (err) { + logger.error('Error getting staff data from Redis:', err); + return null; + } } /** @@ -78,15 +78,15 @@ export async function getStaffData(): Promise { * @param data - The staff data object */ export async function setStaffData(data: any): Promise { - if (!redisClient) { - logger.warn('Redis client not available, skipping setStaffData'); - return; - } - try { - await redisClient.set(STAFF_KEY, JSON.stringify(data)); - } catch (err) { - logger.error('Error setting staff data in Redis:', err); - } + if (!redisClient) { + logger.warn('Redis client not available, skipping setStaffData'); + return; + } + try { + await redisClient.set(STAFF_KEY, JSON.stringify(data)); + } catch (err) { + logger.error('Error setting staff data in Redis:', err); + } } /** @@ -95,39 +95,39 @@ export async function setStaffData(data: any): Promise { * @returns An array of keys */ export async function getAllActivityKeys(): Promise { - if (!redisClient) { - logger.warn('Redis client not available, skipping getAllActivityKeys'); - return []; - } - try { - // Using raw SCAN command since Bun's RedisClient doesn't have a scan method - const keys: string[] = []; - let cursor = '0'; - - do { - // Use send method to execute raw Redis commands - const result = await redisClient.send('SCAN', [ - cursor, - 'MATCH', - `${ACTIVITY_KEY_PREFIX}*`, - 'COUNT', - '100' - ]); - - cursor = result[0]; - const foundKeys = result[1] || []; - - // Add the found keys to our array - keys.push(...foundKeys); - - } while (cursor !== '0'); - - logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`); - return keys; - } catch (err) { - logger.error('Error getting all activity keys from Redis using SCAN:', err); - return []; // Return empty array on error - } + if (!redisClient) { + logger.warn('Redis client not available, skipping getAllActivityKeys'); + return []; + } + try { + // Using raw SCAN command since Bun's RedisClient doesn't have a scan method + const keys: string[] = []; + let cursor = '0'; + + do { + // Use send method to execute raw Redis commands + const result = await redisClient.send('SCAN', [ + cursor, + 'MATCH', + `${ACTIVITY_KEY_PREFIX}*`, + 'COUNT', + '100' + ]); + + cursor = result[0]; + const foundKeys = result[1] || []; + + // Add the found keys to our array + keys.push(...foundKeys); + + } while (cursor !== '0'); + + logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`); + return keys; + } catch (err) { + logger.error('Error getting all activity keys from Redis using SCAN:', err); + return []; // Return empty array on error + } } /** @@ -135,15 +135,15 @@ export async function getAllActivityKeys(): Promise { * @returns The Redis client or null if not initialized */ export function getRedisClient(): RedisClient | null { - return redisClient; + return redisClient; } /** * Closes the Redis connection. */ export async function closeRedisConnection(): Promise { - if (redisClient) { - redisClient.close(); - logger.info('Redis connection closed.'); - } + if (redisClient) { + redisClient.close(); + logger.info('Redis connection closed.'); + } } diff --git a/services/s3-service.ts b/services/s3-service.ts index b6f960d..37c0abb 100644 --- a/services/s3-service.ts +++ b/services/s3-service.ts @@ -20,20 +20,20 @@ const PUBLIC_URL_FILE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').rep let s3Client: S3Client | null = null; if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) { - try { - s3Client = new S3Client({ - accessKeyId: S3_ACCESS_KEY_ID, - secretAccessKey: S3_SECRET_ACCESS_KEY, - bucket: BUCKET_NAME, - endpoint: S3_ENDPOINT, - region: S3_REGION - }); - logger.info('S3 client initialized successfully.'); - } catch (error) { - logger.error('Failed to initialize S3 client:', error); - } + try { + s3Client = new S3Client({ + accessKeyId: S3_ACCESS_KEY_ID, + secretAccessKey: S3_SECRET_ACCESS_KEY, + bucket: BUCKET_NAME, + endpoint: S3_ENDPOINT, + region: S3_REGION + }); + logger.info('S3 client initialized successfully.'); + } catch (error) { + logger.error('Failed to initialize S3 client:', error); + } } else { - logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.'); + logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.'); } /** @@ -44,51 +44,48 @@ if (S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) { * @returns The public URL of the uploaded image or null on error */ export async function uploadImageFromBase64( - base64Data: string, - originalFormat: string, - activityId: string + base64Data: string, + originalFormat: string, + activityId: string ): Promise { - if (!s3Client) { - logger.warn('S3 client not configured. Cannot upload image.'); - return null; - } - if (!base64Data || !originalFormat || !activityId) { - logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId'); - return null; - } + if (!s3Client) { + logger.warn('S3 client not configured. Cannot upload image.'); + return null; + } + if (!base64Data || !originalFormat || !activityId) { + logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId'); + return null; + } - try { - // First decode the base64 image - const imageBuffer = decodeBase64Image(base64Data); - - // Convert to AVIF format with quality 80 using Sharp - const avifBuffer = await sharp(imageBuffer) - .avif({ - quality: 80, - // You can add more AVIF options here if needed - // lossless: false, - // effort: 4, - }) - .toBuffer(); - - // Use .avif extension for the object key - const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`; - - // Using Bun's S3Client file API - const s3File = s3Client.file(objectKey); - - await s3File.write(avifBuffer, { - type: 'image/avif', - acl: 'public-read' - }); - - const publicUrl = constructS3Url(objectKey); - logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`); - return publicUrl; - } catch (error) { - logger.error(`S3 Upload Error for activity ${activityId}:`, error); - return null; - } + try { + // First decode the base64 image + const imageBuffer = decodeBase64Image(base64Data); + // Convert to AVIF format with quality 80 using Sharp + const avifBuffer = await sharp(imageBuffer) + .avif({ + quality: 80, + // You can add more AVIF options here if needed + // lossless: false, + // effort: 4, + }) + .toBuffer(); + // Use .avif extension for the object key + const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.avif`; + // Using Bun's S3Client file API + const s3File = s3Client.file(objectKey); + + await s3File.write(avifBuffer, { + type: 'image/avif', + acl: 'public-read' + }); + + const publicUrl = constructS3Url(objectKey); + logger.info(`Image uploaded to S3 as AVIF: ${publicUrl}`); + return publicUrl; + } catch (error) { + logger.error(`S3 Upload Error for activity ${activityId}:`, error); + return null; + } } /** @@ -97,54 +94,52 @@ export async function uploadImageFromBase64( * @returns A list of object keys */ export async function listS3Objects(prefix: string): Promise { - if (!s3Client) { - logger.warn('S3 client not configured. Cannot list objects.'); - return []; - } + if (!s3Client) { + logger.warn('S3 client not configured. Cannot list objects.'); + return []; + } + + logger.debug(`Listing objects from S3 with prefix: "${prefix}"`); + + try { + const objectKeys: string[] = []; + let isTruncated = true; + let startAfter: string | undefined; - logger.debug(`Listing objects from S3 with prefix: "${prefix}"`); - - try { - const objectKeys: string[] = []; - let isTruncated = true; - let startAfter: string | undefined; - - while (isTruncated) { - // Use Bun's list method with pagination - const result = await s3Client.list({ - prefix, - startAfter, - maxKeys: 1000 - }); - - if (result.contents) { - // Add keys to our array, filtering out "directories" - result.contents.forEach(item => { - if (item.key && !item.key.endsWith('/')) { - objectKeys.push(item.key); - } - }); - - // Get the last key for pagination - if (result.contents?.length > 0) { - startAfter = result.contents[result.contents.length - 1]?.key; - } - } - - isTruncated = result.isTruncated || false; - - // Safety check to prevent infinite loops - if (result.contents?.length === 0) { - break; - } + while (isTruncated) { + // Use Bun's list method with pagination + const result = await s3Client.list({ + prefix, + startAfter, + maxKeys: 1000 + }); + + if (result.contents) { + // Add keys to our array, filtering out "directories" + result.contents.forEach(item => { + if (item.key && !item.key.endsWith('/')) { + objectKeys.push(item.key); + } + }); + // Get the last key for pagination + if (result.contents?.length > 0) { + startAfter = result.contents[result.contents.length - 1]?.key; } - - logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`); - return objectKeys; - } catch (error) { - logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error); - return []; + } + + isTruncated = result.isTruncated || false; + // Safety check to prevent infinite loops + if (result.contents?.length === 0) { + break; + } } + + logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`); + return objectKeys; + } catch (error) { + logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error); + return []; + } } /** @@ -153,47 +148,45 @@ export async function listS3Objects(prefix: string): Promise { * @returns True if successful or partially successful, false on major error */ export async function deleteS3Objects(objectKeysArray: string[]): Promise { - if (!s3Client) { - logger.warn('S3 client not configured. Cannot delete objects.'); - return false; - } - if (!objectKeysArray || objectKeysArray.length === 0) { - logger.info('No objects to delete from S3.'); - return true; - } + if (!s3Client) { + logger.warn('S3 client not configured. Cannot delete objects.'); + return false; + } + if (!objectKeysArray || objectKeysArray.length === 0) { + logger.info('No objects to delete from S3.'); + return true; + } - try { - // With Bun's S3Client, we need to delete objects one by one - // Process in batches of 100 for better performance - const BATCH_SIZE = 100; - let successCount = 0; - let errorCount = 0; - - for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) { - const batch = objectKeysArray.slice(i, i + BATCH_SIZE); - - // Process batch in parallel - const results = await Promise.allSettled( - batch.map(key => s3Client!.delete(key)) - ); - - // Count successes and failures - for (const result of results) { - if (result.status === 'fulfilled') { - successCount++; - } else { - errorCount++; - logger.error(`Failed to delete object: ${result.reason}`); - } - } + try { + // With Bun's S3Client, we need to delete objects one by one + // Process in batches of 100 for better performance + const BATCH_SIZE = 100; + let successCount = 0; + let errorCount = 0; + + for (let i = 0; i < objectKeysArray.length; i += BATCH_SIZE) { + const batch = objectKeysArray.slice(i, i + BATCH_SIZE); + // Process batch in parallel + const results = await Promise.allSettled( + batch.map(key => s3Client!.delete(key)) + ); + // Count successes and failures + for (const result of results) { + if (result.status === 'fulfilled') { + successCount++; + } else { + errorCount++; + logger.error(`Failed to delete object: ${result.reason}`); } - - logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`); - return errorCount === 0; // True if all succeeded - } catch (error) { - logger.error('S3 DeleteObjects Error:', error); - return false; + } } + + logger.info(`Deleted ${successCount} objects from S3. Failed: ${errorCount}`); + return errorCount === 0; // True if all succeeded + } catch (error) { + logger.error('S3 DeleteObjects Error:', error); + return false; + } } /** @@ -202,16 +195,15 @@ export async function deleteS3Objects(objectKeysArray: string[]): Promise svg - { prefix: "data:image/webp;base64,", format: "webp" } - ]; - - for (const marker of markers) { - if (dataUrl.startsWith(marker.prefix)) { - const base64Content = dataUrl.substring(marker.prefix.length); - logger.debug(`Found image of format: ${marker.format}`); - return { base64Content, format: marker.format }; - } - } - - logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "..."); + if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) { return null; + } + + const markers: ImageMarker[] = [ + { prefix: "data:image/png;base64,", format: "png" }, + { prefix: "data:image/jpeg;base64,", format: "jpeg" }, + { prefix: "data:image/jpg;base64,", format: "jpg" }, + { prefix: "data:image/gif;base64,", format: "gif" }, + { prefix: "data:image/svg+xml;base64,", format: "svg" }, // svg+xml -> svg + { prefix: "data:image/webp;base64,", format: "webp" } + ]; + + for (const marker of markers) { + if (dataUrl.startsWith(marker.prefix)) { + const base64Content = dataUrl.substring(marker.prefix.length); + logger.debug(`Found image of format: ${marker.format}`); + return { base64Content, format: marker.format }; + } + } + + logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "..."); + return null; } /** @@ -56,8 +56,8 @@ export function extractBase64Image(dataUrl: string): ImageInfo | null { * @returns {Uint8Array} The decoded binary data */ export function decodeBase64Image(base64String: string): Uint8Array { - // Bun uses Node.js Buffer API and has highly optimized Buffer operations - return Buffer.from(base64String, 'base64'); + // Bun uses Node.js Buffer API and has highly optimized Buffer operations + return Buffer.from(base64String, 'base64'); } /** @@ -67,8 +67,8 @@ export function decodeBase64Image(base64String: string): Uint8Array { * @returns {Uint8Array|null} The decoded image data or null if invalid */ export function dataUrlToBuffer(dataUrl: string): Uint8Array | null { - const imageInfo = extractBase64Image(dataUrl); - if (!imageInfo) return null; - - return decodeBase64Image(imageInfo.base64Content); + const imageInfo = extractBase64Image(dataUrl); + if (!imageInfo) return null; + + return decodeBase64Image(imageInfo.base64Content); } \ No newline at end of file