JamesFlare/dsas-cca-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: redis cache and detach image into s3

Browse Source
This commit is contained in:
JamesFlare1212
2025-05-09 19:43:01 -04:00
parent 99d1ee0a1e
commit f7252345f3
14 changed files with 2302 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
engage-api/get-activity.mjs
View File

@@ -1,9 +1,9 @@
// get-activity.mjs
import axios from 'axios';
import fs from 'fs/promises'; // Using fs.promises directly
import path from 'path';
import { fileURLToPath } from 'url';
import { logger } from '../utils/logger.mjs';
// --- Replicating __dirname for ESM ---
const __filename = fileURLToPath(import.meta.url);
@@ -25,21 +25,21 @@ class AuthenticationError extends Error {
// --- Cookie Cache Helper Functions ---
async function loadCachedCookie() {
if (_inMemoryCookie) {
console.log("Using in-memory cached cookie.");
logger.debug("Using in-memory cached cookie.");
return _inMemoryCookie;
}
try {
const cookieFromFile = await fs.readFile(COOKIE_FILE_PATH, 'utf8');
if (cookieFromFile) {
_inMemoryCookie = cookieFromFile;
console.log("Loaded cookie from file cache.");
logger.debug("Loaded cookie from file cache.");
return _inMemoryCookie;
}
} catch (err) {
if (err.code === 'ENOENT') {
console.log("Cookie cache file not found. No cached cookie loaded.");
logger.debug("Cookie cache file not found. No cached cookie loaded.");
} else {
console.warn("Error loading cookie from file:", err.message);
logger.warn("Error loading cookie from file:", err.message);
}
}
return null;
@@ -47,15 +47,15 @@ async function loadCachedCookie() {
async function saveCookieToCache(cookieString) {
if (!cookieString) {
console.warn("Attempted to save an empty or null cookie. Aborting save.");
logger.warn("Attempted to save an empty or null cookie. Aborting save.");
return;
}
_inMemoryCookie = cookieString;
try {
await fs.writeFile(COOKIE_FILE_PATH, cookieString, 'utf8');
console.log("Cookie saved to file cache.");
logger.debug("Cookie saved to file cache.");
} catch (err) {
console.error("Error saving cookie to file:", err.message);
logger.error("Error saving cookie to file:", err.message);
}
}
@@ -63,19 +63,19 @@ async function clearCookieCache() {
_inMemoryCookie = null;
try {
await fs.unlink(COOKIE_FILE_PATH);
console.log("Cookie cache file deleted.");
logger.debug("Cookie cache file deleted.");
} catch (err) {
if (err.code !== 'ENOENT') {
console.error("Error deleting cookie file:", err.message);
logger.error("Error deleting cookie file:", err.message);
} else {
console.log("Cookie cache file did not exist, no need to delete.");
logger.debug("Cookie cache file did not exist, no need to delete.");
}
}
}
async function testCookieValidity(cookieString) {
if (!cookieString) return false;
console.log("Testing cookie validity...");
logger.debug("Testing cookie validity...");
try {
const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
const headers = {
@@ -85,14 +85,14 @@ async function testCookieValidity(cookieString) {
};
const payload = { "activityID": "3350" };
await axios.post(url, payload, { headers, timeout: 10000 });
console.log("Cookie test successful (API responded 2xx). Cookie is valid.");
logger.debug("Cookie test successful (API responded 2xx). Cookie is valid.");
return true;
} catch (error) {
console.warn("Cookie validity test failed.");
logger.warn("Cookie validity test failed.");
if (error.response) {
console.warn(`Cookie test API response status: ${error.response.status}. Cookie is likely invalid or expired.`);
logger.warn(`Cookie test API response status: ${error.response.status}. Cookie is likely invalid or expired.`);
} else {
console.warn(`Cookie test failed due to network or other error: ${error.message}`);
logger.warn(`Cookie test failed due to network or other error: ${error.message}`);
}
return false;
}
@@ -109,14 +109,14 @@ async function getSessionId() {
if (setCookieHeader && setCookieHeader.length > 0) {
const sessionIdCookie = setCookieHeader.find(cookie => cookie.trim().startsWith('ASP.NET_SessionId='));
if (sessionIdCookie) {
console.log('Debugging - ASP.NET_SessionId created');
logger.debug('ASP.NET_SessionId created');
return sessionIdCookie.split(';')[0];
}
}
console.error("No ASP.NET_SessionId cookie found in Set-Cookie header.");
logger.error("No ASP.NET_SessionId cookie found in Set-Cookie header.");
return null;
} catch (error) {
console.error(`Error in getSessionId: ${error.response ? `${error.response.status} - ${error.response.statusText}` : error.message}`);
logger.error(`Error in getSessionId: ${error.response ? `${error.response.status} - ${error.response.statusText}` : error.message}`);
throw error;
}
}
@@ -133,7 +133,7 @@ async function getMSAUTH(sessionId, userName, userPwd, templateFilePath) {
'User-Agent': 'Mozilla/5.0 (Node.js DSAS-CCA get-activity Module)',
'Referer': 'https://engage.nkcswx.cn/Login.aspx'
};
console.log('Debugging - Getting .ASPXFORMSAUTH');
logger.debug('Getting .ASPXFORMSAUTH');
const response = await axios.post(url, postData, {
headers, maxRedirects: 0,
validateStatus: (status) => status >= 200 && status < 400
@@ -153,21 +153,21 @@ async function getMSAUTH(sessionId, userName, userPwd, templateFilePath) {
}
}
if (formsAuthCookieValue) {
console.log('Debugging - .ASPXFORMSAUTH cookie obtained.');
logger.debug('.ASPXFORMSAUTH cookie obtained.');
return formsAuthCookieValue;
} else {
console.error("No valid .ASPXFORMSAUTH cookie found. Headers:", setCookieHeader || "none");
logger.error("No valid .ASPXFORMSAUTH cookie found. Headers:", setCookieHeader || "none");
return null;
}
} catch (error) {
if (error.code === 'ENOENT') console.error(`Error: Template file '${templateFilePath}' not found.`);
else console.error(`Error in getMSAUTH: ${error.message}`);
if (error.code === 'ENOENT') logger.error(`Error: Template file '${templateFilePath}' not found.`);
else logger.error(`Error in getMSAUTH: ${error.message}`);
throw error;
}
}
async function getCompleteCookies(userName, userPwd, templateFilePath) {
console.log('Debugging - Attempting to get complete cookie string (login process).');
logger.debug('Attempting to get complete cookie string (login process).');
const sessionId = await getSessionId();
if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId.");
@@ -195,24 +195,25 @@ async function getActivityDetailsRaw(activityId, cookies, maxRetries = 3, timeou
if (outerData && typeof outerData.d === 'string') {
const innerData = JSON.parse(outerData.d);
if (innerData.isError) {
console.warn(`API reported isError:true for activity ${activityId}.`);
logger.warn(`API reported isError:true for activity ${activityId}.`);
return null;
}
return response.data;
} else {
console.error(`Unexpected API response structure for activity ${activityId}.`);
logger.error(`Unexpected API response structure for activity ${activityId}.`);
}
} catch (error) {
if (error.response && (error.response.status === 403 || error.response.status === 401)) {
console.warn(`Authentication error (${error.response.status}) while fetching activity ${activityId}. Cookie may be invalid.`);
// Check if response status is in 4xx range (400-499) to trigger auth error
if (error.response && error.response.status >= 400 && error.response.status < 500) {
logger.warn(`Authentication error (${error.response.status}) while fetching activity ${activityId}. Cookie may be invalid.`);
throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status);
}
console.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`);
if (error.response) {
console.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`);
}
if (attempt === maxRetries - 1) {
console.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`);
throw error;
}
await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
@@ -241,27 +242,27 @@ export async function fetchActivityData(activityId, userName, userPwd, templateF
if (currentCookie) {
const isValid = await testCookieValidity(currentCookie);
if (!isValid) {
console.log("Cached cookie test failed or cookie expired. Clearing cache.");
logger.info("Cached cookie test failed or cookie expired. Clearing cache.");
await clearCookieCache();
currentCookie = null;
} else {
console.log("Using valid cached cookie.");
logger.info("Using valid cached cookie.");
}
}
if (!currentCookie) {
console.log(forceLogin ? "Forcing new login." : "No valid cached cookie found or cache bypassed. Attempting login...");
logger.info(forceLogin ? "Forcing new login." : "No valid cached cookie found or cache bypassed. Attempting login...");
try {
currentCookie = await getCompleteCookies(userName, userPwd, path.resolve(__dirname, templateFileName));
await saveCookieToCache(currentCookie);
} catch (loginError) {
console.error(`Login process failed: ${loginError.message}`);
logger.error(`Login process failed: ${loginError.message}`);
return null;
}
}
if (!currentCookie) {
console.error("Critical: No cookie available after login attempt. Cannot fetch activity data.");
logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data.");
return null;
}
@@ -271,32 +272,32 @@ export async function fetchActivityData(activityId, userName, userPwd, templateF
const parsedOuter = JSON.parse(rawActivityDetailsString);
return JSON.parse(parsedOuter.d);
}
console.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`);
logger.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`);
return null;
} catch (error) {
if (error instanceof AuthenticationError) {
console.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`);
logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`);
await clearCookieCache();
try {
console.log("Attempting re-login due to authentication failure...");
logger.info("Attempting re-login due to authentication failure...");
currentCookie = await getCompleteCookies(userName, userPwd, path.resolve(__dirname, templateFileName));
await saveCookieToCache(currentCookie);
console.log("Re-login successful. Retrying request for activity details once...");
logger.info("Re-login successful. Retrying request for activity details once...");
const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie);
if (rawActivityDetailsStringRetry) {
const parsedOuterRetry = JSON.parse(rawActivityDetailsStringRetry);
return JSON.parse(parsedOuterRetry.d);
}
console.warn(`Still no details for activity ${activityId} after re-login and retry.`);
logger.warn(`Still no details for activity ${activityId} after re-login and retry.`);
return null;
} catch (retryLoginOrFetchError) {
console.error(`Error during re-login or retry fetch for activity ${activityId}: ${retryLoginOrFetchError.message}`);
logger.error(`Error during re-login or retry fetch for activity ${activityId}: ${retryLoginOrFetchError.message}`);
return null;
}
} else {
console.error(`Failed to fetch activity data for ${activityId} due to non-authentication error: ${error.message}`);
logger.error(`Failed to fetch activity data for ${activityId} due to non-authentication error: ${error.message}`);
return null;
}
}

64
engage-api/struct-activity.mjs
View File

@@ -1,4 +1,5 @@
// struct-activity.mjs
import { logger } from '../utils/logger.mjs';
let clubSchema = {
academicYear: null,
@@ -84,21 +85,34 @@ async function applyFields(field, structuredActivityData) {
structuredActivityData.duration.isRecurringWeekly = true;
break;
default:
//console.log(`No matching case for field: fID=${field.fID}, fType=${field.fType}`);
logger.debug(`No matching case for field: fID=${field.fID}, fType=${field.fType}`);
break;
}
}
async function postProcess(structuredActivityData) {
structuredActivityData.description = structuredActivityData.description.replaceAll("<br/>","\n");
structuredActivityData.description = structuredActivityData.description.replaceAll("\u000B","\v");
if (structuredActivityData.name.search("Student-led") != -1) {
structuredActivityData.isStudentLed = true;
} else {
structuredActivityData.isStudentLed = false;
}
const grades = structuredActivityData.schedule.match(/G(\d+)-(\d+)/);
structuredActivityData.grades.min = grades[1];
structuredActivityData.grades.max = grades[2];
try {
let grades = structuredActivityData.schedule.match(/G(\d+)-(\d+)/) ||
structuredActivityData.schedule.match(/KG(\d+)-KG(\d+)/);
if (!grades || grades.length < 3) {
throw new Error('Invalid grade format in schedule');
}
structuredActivityData.grades.min = grades[1];
structuredActivityData.grades.max = grades[2];
} catch (error) {
logger.error(`Failed to parse grades: ${error.message}`);
structuredActivityData.grades.min = null;
structuredActivityData.grades.max = null;
}
}
export async function structActivityData(rawActivityData) {
@@ -110,35 +124,39 @@ export async function structActivityData(rawActivityData) {
for (let i = 0; i < rowObject.fields.length; i++) {
const field = rowObject.fields[i];
// Optimize: no fData, just skip
if (field.fData == null && field.fData == "") { continue; }
if (!field.fData) { continue; }
// Process hard cases first
if (field.fData == "Description") {
structuredActivityData.description = rowObject.fields[i + 1].fData;
if (i + 1 < rowObject.fields.length) {
structuredActivityData.description = rowObject.fields[i + 1].fData;
}
continue;
} else if (field.fData == "Name To Appear On Reports"){
let staffForReports = rowObject.fields[i + 1].fData.split(", ");
structuredActivityData.staffForReports = staffForReports;
} else if (field.fData == "Name To Appear On Reports") {
if (i + 1 < rowObject.fields.length) {
let staffForReports = rowObject.fields[i + 1].fData.split(", ");
structuredActivityData.staffForReports = staffForReports;
}
} else if (field.fData == "Upload Photo") {
structuredActivityData.photo = rowObject.fields[i + 1].fData;
if (i + 1 < rowObject.fields.length) {
structuredActivityData.photo = rowObject.fields[i + 1].fData;
}
} else if (field.fData == "Poor Weather Plan") {
structuredActivityData.poorWeatherPlan = rowObject.fields[i + 1].fData;
if (i + 1 < rowObject.fields.length) {
structuredActivityData.poorWeatherPlan = rowObject.fields[i + 1].fData;
}
} else if (field.fData == "Activity Runs From") {
if (rowObject.fields[i + 4].fData == "Recurring Weekly") {
structuredActivityData.duration.isRecurringWeekly = true;
} else {
structuredActivityData.duration.isRecurringWeekly = false;
if (i + 4 < rowObject.fields.length) {
structuredActivityData.duration.isRecurringWeekly =
rowObject.fields[i + 4].fData == "Recurring Weekly";
}
} else if (field.fData == "Is Pre Sign-up") {
if (rowObject.fields[i + 1].fData == "") {
structuredActivityData.isPreSignup = false;
} else {
structuredActivityData.isPreSignup = true;
if (i + 1 < rowObject.fields.length) {
structuredActivityData.isPreSignup = rowObject.fields[i + 1].fData !== "";
}
} else if (field.fData == "Semester Cost") {
if (rowObject.fields[i + 1].fData == "") {
structuredActivityData.semesterCost = null;
} else {
structuredActivityData.semesterCost = rowObject.fields[i + 1].fData
if (i + 1 < rowObject.fields.length) {
structuredActivityData.semesterCost =
rowObject.fields[i + 1].fData === "" ? null : rowObject.fields[i + 1].fData;
}
} else {
// Pass any other easy cases to helper function