From f7252345f354e9c6f3a667781ac99b18cde97634 Mon Sep 17 00:00:00 2001 From: JamesFlare1212 Date: Fri, 9 May 2025 19:43:01 -0400 Subject: [PATCH] feat: redis cache and detach image into s3 --- .gitignore | 3 +- docker-compose.yaml | 27 +- engage-api/get-activity.mjs | 89 +-- engage-api/struct-activity.mjs | 64 +- example.env | 14 +- main.js | 128 ---- main.mjs | 289 ++++++++ package.json | 10 +- pnpm-lock.yaml | 1277 ++++++++++++++++++++++++++++++++ services/cache-manager.mjs | 209 ++++++ services/redis-service.mjs | 135 ++++ services/s3-service.mjs | 186 +++++ utils/image-processor.mjs | 43 ++ utils/logger.mjs | 30 + 14 files changed, 2302 insertions(+), 202 deletions(-) delete mode 100644 main.js create mode 100644 main.mjs create mode 100644 services/cache-manager.mjs create mode 100644 services/redis-service.mjs create mode 100644 services/s3-service.mjs create mode 100644 utils/image-processor.mjs create mode 100644 utils/logger.mjs diff --git a/.gitignore b/.gitignore index 441104d..36ecc09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules nkcs-engage.cookie.txt -.env \ No newline at end of file +.env +redis_data \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index b160fbc..a390896 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,5 @@ services: - dsas-cca-backend: + app: build: context: . dockerfile: Dockerfile @@ -8,4 +8,27 @@ services: - "${PORT}:${PORT}" env_file: - .env - restart: unless-stopped \ No newline at end of file + restart: unless-stopped + depends_on: + - redis + networks: + - cca_network + + redis: + image: "redis:7.2-alpine" + container_name: dsas-cca-redis + command: redis-server --requirepass "dsas-cca" + volumes: + - ./redis_data:/data + restart: unless-stopped + networks: + - cca_network + healthcheck: + test: ["CMD", "redis-cli", "-a", "dsas-cca", "ping"] + interval: 10s + timeout: 5s + retries: 5 + +networks: + cca_network: + driver: bridge diff --git a/engage-api/get-activity.mjs b/engage-api/get-activity.mjs index 253cc1d..3348c6a 100644 --- a/engage-api/get-activity.mjs +++ b/engage-api/get-activity.mjs @@ -1,9 +1,9 @@ // get-activity.mjs - import axios from 'axios'; import fs from 'fs/promises'; // Using fs.promises directly import path from 'path'; import { fileURLToPath } from 'url'; +import { logger } from '../utils/logger.mjs'; // --- Replicating __dirname for ESM --- const __filename = fileURLToPath(import.meta.url); @@ -25,21 +25,21 @@ class AuthenticationError extends Error { // --- Cookie Cache Helper Functions --- async function loadCachedCookie() { if (_inMemoryCookie) { - console.log("Using in-memory cached cookie."); + logger.debug("Using in-memory cached cookie."); return _inMemoryCookie; } try { const cookieFromFile = await fs.readFile(COOKIE_FILE_PATH, 'utf8'); if (cookieFromFile) { _inMemoryCookie = cookieFromFile; - console.log("Loaded cookie from file cache."); + logger.debug("Loaded cookie from file cache."); return _inMemoryCookie; } } catch (err) { if (err.code === 'ENOENT') { - console.log("Cookie cache file not found. No cached cookie loaded."); + logger.debug("Cookie cache file not found. No cached cookie loaded."); } else { - console.warn("Error loading cookie from file:", err.message); + logger.warn("Error loading cookie from file:", err.message); } } return null; @@ -47,15 +47,15 @@ async function loadCachedCookie() { async function saveCookieToCache(cookieString) { if (!cookieString) { - console.warn("Attempted to save an empty or null cookie. Aborting save."); + logger.warn("Attempted to save an empty or null cookie. Aborting save."); return; } _inMemoryCookie = cookieString; try { await fs.writeFile(COOKIE_FILE_PATH, cookieString, 'utf8'); - console.log("Cookie saved to file cache."); + logger.debug("Cookie saved to file cache."); } catch (err) { - console.error("Error saving cookie to file:", err.message); + logger.error("Error saving cookie to file:", err.message); } } @@ -63,19 +63,19 @@ async function clearCookieCache() { _inMemoryCookie = null; try { await fs.unlink(COOKIE_FILE_PATH); - console.log("Cookie cache file deleted."); + logger.debug("Cookie cache file deleted."); } catch (err) { if (err.code !== 'ENOENT') { - console.error("Error deleting cookie file:", err.message); + logger.error("Error deleting cookie file:", err.message); } else { - console.log("Cookie cache file did not exist, no need to delete."); + logger.debug("Cookie cache file did not exist, no need to delete."); } } } async function testCookieValidity(cookieString) { if (!cookieString) return false; - console.log("Testing cookie validity..."); + logger.debug("Testing cookie validity..."); try { const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails'; const headers = { @@ -85,14 +85,14 @@ async function testCookieValidity(cookieString) { }; const payload = { "activityID": "3350" }; await axios.post(url, payload, { headers, timeout: 10000 }); - console.log("Cookie test successful (API responded 2xx). Cookie is valid."); + logger.debug("Cookie test successful (API responded 2xx). Cookie is valid."); return true; } catch (error) { - console.warn("Cookie validity test failed."); + logger.warn("Cookie validity test failed."); if (error.response) { - console.warn(`Cookie test API response status: ${error.response.status}. Cookie is likely invalid or expired.`); + logger.warn(`Cookie test API response status: ${error.response.status}. Cookie is likely invalid or expired.`); } else { - console.warn(`Cookie test failed due to network or other error: ${error.message}`); + logger.warn(`Cookie test failed due to network or other error: ${error.message}`); } return false; } @@ -109,14 +109,14 @@ async function getSessionId() { if (setCookieHeader && setCookieHeader.length > 0) { const sessionIdCookie = setCookieHeader.find(cookie => cookie.trim().startsWith('ASP.NET_SessionId=')); if (sessionIdCookie) { - console.log('Debugging - ASP.NET_SessionId created'); + logger.debug('ASP.NET_SessionId created'); return sessionIdCookie.split(';')[0]; } } - console.error("No ASP.NET_SessionId cookie found in Set-Cookie header."); + logger.error("No ASP.NET_SessionId cookie found in Set-Cookie header."); return null; } catch (error) { - console.error(`Error in getSessionId: ${error.response ? `${error.response.status} - ${error.response.statusText}` : error.message}`); + logger.error(`Error in getSessionId: ${error.response ? `${error.response.status} - ${error.response.statusText}` : error.message}`); throw error; } } @@ -133,7 +133,7 @@ async function getMSAUTH(sessionId, userName, userPwd, templateFilePath) { 'User-Agent': 'Mozilla/5.0 (Node.js DSAS-CCA get-activity Module)', 'Referer': 'https://engage.nkcswx.cn/Login.aspx' }; - console.log('Debugging - Getting .ASPXFORMSAUTH'); + logger.debug('Getting .ASPXFORMSAUTH'); const response = await axios.post(url, postData, { headers, maxRedirects: 0, validateStatus: (status) => status >= 200 && status < 400 @@ -153,21 +153,21 @@ async function getMSAUTH(sessionId, userName, userPwd, templateFilePath) { } } if (formsAuthCookieValue) { - console.log('Debugging - .ASPXFORMSAUTH cookie obtained.'); + logger.debug('.ASPXFORMSAUTH cookie obtained.'); return formsAuthCookieValue; } else { - console.error("No valid .ASPXFORMSAUTH cookie found. Headers:", setCookieHeader || "none"); + logger.error("No valid .ASPXFORMSAUTH cookie found. Headers:", setCookieHeader || "none"); return null; } } catch (error) { - if (error.code === 'ENOENT') console.error(`Error: Template file '${templateFilePath}' not found.`); - else console.error(`Error in getMSAUTH: ${error.message}`); + if (error.code === 'ENOENT') logger.error(`Error: Template file '${templateFilePath}' not found.`); + else logger.error(`Error in getMSAUTH: ${error.message}`); throw error; } } async function getCompleteCookies(userName, userPwd, templateFilePath) { - console.log('Debugging - Attempting to get complete cookie string (login process).'); + logger.debug('Attempting to get complete cookie string (login process).'); const sessionId = await getSessionId(); if (!sessionId) throw new Error("Login failed: Could not obtain ASP.NET_SessionId."); @@ -195,24 +195,25 @@ async function getActivityDetailsRaw(activityId, cookies, maxRetries = 3, timeou if (outerData && typeof outerData.d === 'string') { const innerData = JSON.parse(outerData.d); if (innerData.isError) { - console.warn(`API reported isError:true for activity ${activityId}.`); + logger.warn(`API reported isError:true for activity ${activityId}.`); return null; } return response.data; } else { - console.error(`Unexpected API response structure for activity ${activityId}.`); + logger.error(`Unexpected API response structure for activity ${activityId}.`); } } catch (error) { - if (error.response && (error.response.status === 403 || error.response.status === 401)) { - console.warn(`Authentication error (${error.response.status}) while fetching activity ${activityId}. Cookie may be invalid.`); + // Check if response status is in 4xx range (400-499) to trigger auth error + if (error.response && error.response.status >= 400 && error.response.status < 500) { + logger.warn(`Authentication error (${error.response.status}) while fetching activity ${activityId}. Cookie may be invalid.`); throw new AuthenticationError(`Received ${error.response.status} for activity ${activityId}`, error.response.status); } - console.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`); + logger.error(`Attempt ${attempt + 1}/${maxRetries} for activity ${activityId} failed: ${error.message}`); if (error.response) { - console.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`); + logger.error(`Status: ${error.response.status}, Data (getActivityDetailsRaw): ${ String(error.response.data).slice(0,100)}...`); } if (attempt === maxRetries - 1) { - console.error(`All ${maxRetries} retries failed for activity ${activityId}.`); + logger.error(`All ${maxRetries} retries failed for activity ${activityId}.`); throw error; } await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1))); @@ -241,27 +242,27 @@ export async function fetchActivityData(activityId, userName, userPwd, templateF if (currentCookie) { const isValid = await testCookieValidity(currentCookie); if (!isValid) { - console.log("Cached cookie test failed or cookie expired. Clearing cache."); + logger.info("Cached cookie test failed or cookie expired. Clearing cache."); await clearCookieCache(); currentCookie = null; } else { - console.log("Using valid cached cookie."); + logger.info("Using valid cached cookie."); } } if (!currentCookie) { - console.log(forceLogin ? "Forcing new login." : "No valid cached cookie found or cache bypassed. Attempting login..."); + logger.info(forceLogin ? "Forcing new login." : "No valid cached cookie found or cache bypassed. Attempting login..."); try { currentCookie = await getCompleteCookies(userName, userPwd, path.resolve(__dirname, templateFileName)); await saveCookieToCache(currentCookie); } catch (loginError) { - console.error(`Login process failed: ${loginError.message}`); + logger.error(`Login process failed: ${loginError.message}`); return null; } } if (!currentCookie) { - console.error("Critical: No cookie available after login attempt. Cannot fetch activity data."); + logger.error("Critical: No cookie available after login attempt. Cannot fetch activity data."); return null; } @@ -271,32 +272,32 @@ export async function fetchActivityData(activityId, userName, userPwd, templateF const parsedOuter = JSON.parse(rawActivityDetailsString); return JSON.parse(parsedOuter.d); } - console.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`); + logger.warn(`No data returned from getActivityDetailsRaw for activity ${activityId}, but no authentication error was thrown.`); return null; } catch (error) { if (error instanceof AuthenticationError) { - console.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`); + logger.warn(`Initial fetch failed with AuthenticationError (Status: ${error.status}). Cookie was likely invalid. Attempting re-login and one retry.`); await clearCookieCache(); try { - console.log("Attempting re-login due to authentication failure..."); + logger.info("Attempting re-login due to authentication failure..."); currentCookie = await getCompleteCookies(userName, userPwd, path.resolve(__dirname, templateFileName)); await saveCookieToCache(currentCookie); - console.log("Re-login successful. Retrying request for activity details once..."); + logger.info("Re-login successful. Retrying request for activity details once..."); const rawActivityDetailsStringRetry = await getActivityDetailsRaw(activityId, currentCookie); if (rawActivityDetailsStringRetry) { const parsedOuterRetry = JSON.parse(rawActivityDetailsStringRetry); return JSON.parse(parsedOuterRetry.d); } - console.warn(`Still no details for activity ${activityId} after re-login and retry.`); + logger.warn(`Still no details for activity ${activityId} after re-login and retry.`); return null; } catch (retryLoginOrFetchError) { - console.error(`Error during re-login or retry fetch for activity ${activityId}: ${retryLoginOrFetchError.message}`); + logger.error(`Error during re-login or retry fetch for activity ${activityId}: ${retryLoginOrFetchError.message}`); return null; } } else { - console.error(`Failed to fetch activity data for ${activityId} due to non-authentication error: ${error.message}`); + logger.error(`Failed to fetch activity data for ${activityId} due to non-authentication error: ${error.message}`); return null; } } diff --git a/engage-api/struct-activity.mjs b/engage-api/struct-activity.mjs index 7590f10..3f65438 100644 --- a/engage-api/struct-activity.mjs +++ b/engage-api/struct-activity.mjs @@ -1,4 +1,5 @@ // struct-activity.mjs +import { logger } from '../utils/logger.mjs'; let clubSchema = { academicYear: null, @@ -84,21 +85,34 @@ async function applyFields(field, structuredActivityData) { structuredActivityData.duration.isRecurringWeekly = true; break; default: - //console.log(`No matching case for field: fID=${field.fID}, fType=${field.fType}`); + logger.debug(`No matching case for field: fID=${field.fID}, fType=${field.fType}`); break; } } async function postProcess(structuredActivityData) { structuredActivityData.description = structuredActivityData.description.replaceAll("
","\n"); + structuredActivityData.description = structuredActivityData.description.replaceAll("\u000B","\v"); if (structuredActivityData.name.search("Student-led") != -1) { structuredActivityData.isStudentLed = true; } else { structuredActivityData.isStudentLed = false; } - const grades = structuredActivityData.schedule.match(/G(\d+)-(\d+)/); - structuredActivityData.grades.min = grades[1]; - structuredActivityData.grades.max = grades[2]; + try { + let grades = structuredActivityData.schedule.match(/G(\d+)-(\d+)/) || + structuredActivityData.schedule.match(/KG(\d+)-KG(\d+)/); + + if (!grades || grades.length < 3) { + throw new Error('Invalid grade format in schedule'); + } + + structuredActivityData.grades.min = grades[1]; + structuredActivityData.grades.max = grades[2]; + } catch (error) { + logger.error(`Failed to parse grades: ${error.message}`); + structuredActivityData.grades.min = null; + structuredActivityData.grades.max = null; + } } export async function structActivityData(rawActivityData) { @@ -110,35 +124,39 @@ export async function structActivityData(rawActivityData) { for (let i = 0; i < rowObject.fields.length; i++) { const field = rowObject.fields[i]; // Optimize: no fData, just skip - if (field.fData == null && field.fData == "") { continue; } + if (!field.fData) { continue; } // Process hard cases first if (field.fData == "Description") { - structuredActivityData.description = rowObject.fields[i + 1].fData; + if (i + 1 < rowObject.fields.length) { + structuredActivityData.description = rowObject.fields[i + 1].fData; + } continue; - } else if (field.fData == "Name To Appear On Reports"){ - let staffForReports = rowObject.fields[i + 1].fData.split(", "); - structuredActivityData.staffForReports = staffForReports; + } else if (field.fData == "Name To Appear On Reports") { + if (i + 1 < rowObject.fields.length) { + let staffForReports = rowObject.fields[i + 1].fData.split(", "); + structuredActivityData.staffForReports = staffForReports; + } } else if (field.fData == "Upload Photo") { - structuredActivityData.photo = rowObject.fields[i + 1].fData; + if (i + 1 < rowObject.fields.length) { + structuredActivityData.photo = rowObject.fields[i + 1].fData; + } } else if (field.fData == "Poor Weather Plan") { - structuredActivityData.poorWeatherPlan = rowObject.fields[i + 1].fData; + if (i + 1 < rowObject.fields.length) { + structuredActivityData.poorWeatherPlan = rowObject.fields[i + 1].fData; + } } else if (field.fData == "Activity Runs From") { - if (rowObject.fields[i + 4].fData == "Recurring Weekly") { - structuredActivityData.duration.isRecurringWeekly = true; - } else { - structuredActivityData.duration.isRecurringWeekly = false; + if (i + 4 < rowObject.fields.length) { + structuredActivityData.duration.isRecurringWeekly = + rowObject.fields[i + 4].fData == "Recurring Weekly"; } } else if (field.fData == "Is Pre Sign-up") { - if (rowObject.fields[i + 1].fData == "") { - structuredActivityData.isPreSignup = false; - } else { - structuredActivityData.isPreSignup = true; + if (i + 1 < rowObject.fields.length) { + structuredActivityData.isPreSignup = rowObject.fields[i + 1].fData !== ""; } } else if (field.fData == "Semester Cost") { - if (rowObject.fields[i + 1].fData == "") { - structuredActivityData.semesterCost = null; - } else { - structuredActivityData.semesterCost = rowObject.fields[i + 1].fData + if (i + 1 < rowObject.fields.length) { + structuredActivityData.semesterCost = + rowObject.fields[i + 1].fData === "" ? null : rowObject.fields[i + 1].fData; } } else { // Pass any other easy cases to helper function diff --git a/example.env b/example.env index e253c99..2427ca0 100644 --- a/example.env +++ b/example.env @@ -2,4 +2,16 @@ API_USERNAME= API_PASSWORD= PORT=3000 FIXED_STAFF_ACTIVITY_ID=7095 -ALLOWED_ORIGINS=* \ No newline at end of file +ALLOWED_ORIGINS=* +S3_ENDPOINT= +S3_BUCKET_NAME= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= +S3_REGION= +S3_PUBLIC_URL_PREFIX=files +REDIS_URL=redis://:dsas-cca@redis:6379 +MAX_ACTIVITY_ID_SCAN=9999 +CONCURRENT_API_CALLS=16 +STAFF_UPDATE_INTERVAL_MINS=360 +CLUB_UPDATE_INTERVAL_MINS=360 +LOG_LEVEL=info # Example: 'debug', 'info', 'warn', 'error' \ No newline at end of file diff --git a/main.js b/main.js deleted file mode 100644 index f840fbf..0000000 --- a/main.js +++ /dev/null @@ -1,128 +0,0 @@ -// main.js -import express from 'express'; -import dotenv from 'dotenv'; -import cors from 'cors'; // Import the cors middleware -import { fetchActivityData } from './engage-api/get-activity.mjs'; -import { structActivityData } from './engage-api/struct-activity.mjs'; -import { structStaffData } from './engage-api/struct-staff.mjs'; - -// Load environment variables from .env file -dotenv.config(); - -// --- Configuration --- -const USERNAME = process.env.API_USERNAME; -const PASSWORD = process.env.API_PASSWORD; -const PORT = process.env.PORT || 3000; -const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID || '3350'; -const allowedOriginsEnv = process.env.ALLOWED_ORIGINS || '*'; - -let corsOptions; - -if (allowedOriginsEnv === '*') { - corsOptions = { origin: '*' }; -} else { - // If ALLOWED_ORIGINS is a comma-separated list, split it into an array - const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim()); - corsOptions = { - origin: function (origin, callback) { - // Allow requests with no origin (like mobile apps or curl requests) - if (!origin) return callback(null, true); - if (originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) { - callback(null, true); - } else { - callback(new Error('Not allowed by CORS')); - } - } - }; -} - -// --- Initialize Express App --- -const app = express(); - -// Apply CORS middleware globally FIRST -// This will add the 'Access-Control-Allow-Origin' header to all responses -app.use(cors(corsOptions)); - -// Middleware to parse JSON request bodies -app.use(express.json()); - -// --- API Endpoints --- - -// GET Endpoint: Welcome message -app.get('/', (req, res) => { - res.send('Welcome to the DSAS CCA API!

\ - API Endpoints:
\ - GET /v1/activity/:activityId (ID must be 1-4 digits)
\ - GET /v1/staffs'); -}); - -// GET Endpoint: Fetch structured activity data by ID -app.get('/v1/activity/:activityId', async (req, res) => { - let { activityId } = req.params; - - if (!/^\d{1,4}$/.test(activityId)) { - return res.status(400).json({ - error: 'Invalid Activity ID format. Activity ID must be 1 to 4 digits (e.g., 1, 0001, 9999).' - }); - } - - if (!USERNAME || !PASSWORD) { - console.error('API username or password not configured. Check .env file or environment variables.'); - return res.status(500).json({ error: 'Server configuration error.' }); - } - - console.log(`Workspaceing activity details for activity ID: ${activityId}`); - try { - const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD); - if (activityJson) { - const structuredActivity = await structActivityData(activityJson); - res.json(structuredActivity); - } else { - res.status(404).json({ error: `Could not retrieve details for activity ${activityId}.` }); - } - } catch (error) { - console.error(`Error fetching activity ${activityId}:`, error); - res.status(500).json({ error: 'An internal server error occurred while fetching activity data.' }); - } -}); - -// GET Endpoint: Fetch fixed structured staff data -app.get('/v1/staffs', async (req, res) => { - if (!USERNAME || !PASSWORD) { - console.error('API username or password not configured. Check .env file or environment variables.'); - return res.status(500).json({ error: 'Server configuration error.' }); - } - - console.log(`Workspaceing staff details (using fixed activity ID: ${FIXED_STAFF_ACTIVITY_ID})`); - try { - const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD); - if (activityJson) { - const staffMap = await structStaffData(activityJson); - const staffObject = Object.fromEntries(staffMap); - res.json(staffObject); - } else { - res.status(404).json({ error: `Could not retrieve base data using activity ID ${FIXED_STAFF_ACTIVITY_ID} to get staff details.` }); - } - } catch (error) { - console.error(`Error fetching staff data (for fixed activity ID ${FIXED_STAFF_ACTIVITY_ID}):`, error); - res.status(500).json({ error: 'An internal server error occurred while fetching staff data.' }); - } -}); - -// --- Start the Server --- -app.listen(PORT, () => { - console.log(`Server is running on http://localhost:${PORT}`); - console.log(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`); - console.log('API Endpoints:'); - console.log(` GET /v1/activity/:activityId (ID must be 1-4 digits)`); - console.log(` GET /v1/staffs`); - if (!USERNAME || !PASSWORD) { - console.warn('Warning: API_USERNAME or API_PASSWORD is not set. Please configure them in your .env file or environment variables.'); - } -}); - -// Graceful shutdown -process.on('SIGINT', () => { - console.log('Server shutting down...'); - process.exit(0); -}); \ No newline at end of file diff --git a/main.mjs b/main.mjs new file mode 100644 index 0000000..be52433 --- /dev/null +++ b/main.mjs @@ -0,0 +1,289 @@ +// main.mjs +import express from 'express'; +import dotenv from 'dotenv'; +import cors from 'cors'; +import { fetchActivityData } from './engage-api/get-activity.mjs'; +import { structActivityData } from './engage-api/struct-activity.mjs'; +import { structStaffData } from './engage-api/struct-staff.mjs'; +import { + getActivityData, + setActivityData, + getStaffData, + setStaffData, + getRedisClient, + getAllActivityKeys, // Make sure this is imported + ACTIVITY_KEY_PREFIX // Import the prefix +} from './services/redis-service.mjs'; +import { uploadImageFromBase64 } from './services/s3-service.mjs'; +import { extractBase64Image } from './utils/image-processor.mjs'; +import { + initializeClubCache, + updateStaleClubs, + initializeOrUpdateStaffCache, + cleanupOrphanedS3Images +} from './services/cache-manager.mjs'; +import { logger } from './utils/logger.mjs'; + +dotenv.config(); + +const USERNAME = process.env.API_USERNAME; +const PASSWORD = process.env.API_PASSWORD; +const PORT = process.env.PORT || 3000; +const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID; +const allowedOriginsEnv = process.env.ALLOWED_ORIGINS || '*'; +const CLUB_CHECK_INTERVAL_SECONDS = parseInt(process.env.CLUB_CHECK_INTERVAL_SECONDS || '300', 10); +const STAFF_CHECK_INTERVAL_SECONDS = parseInt(process.env.STAFF_CHECK_INTERVAL_SECONDS || '300', 10); + +let corsOptions; +if (allowedOriginsEnv === '*') { + corsOptions = { origin: '*' }; +} else { + const originsArray = allowedOriginsEnv.split(',').map(origin => origin.trim()); + corsOptions = { + origin: function (origin, callback) { + if (!origin || originsArray.indexOf(origin) !== -1 || originsArray.includes('*')) { + callback(null, true); + } else { + callback(new Error('Not allowed by CORS')); + } + } + }; +} + +const app = express(); +app.use(cors(corsOptions)); +app.use(express.json()); + +// Helper function to process activity data (fetch, struct, S3, cache) for API calls +async function fetchProcessAndStoreActivity(activityId) { + logger.info(`API call: Cache miss or forced fetch for activity ID: ${activityId}.`); + const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD); + + if (!activityJson) { + logger.warn(`API call: No data from engage API for activity ${activityId}. Caching as empty.`); + const emptyData = { lastCheck: new Date().toISOString(), source: 'api-fetch-empty' }; + await setActivityData(activityId, emptyData); + return { data: emptyData, status: 404 }; + } + + let structuredActivity = await structActivityData(activityJson); + if (structuredActivity && structuredActivity.photo && structuredActivity.photo.startsWith('data:image')) { + const imageInfo = extractBase64Image(structuredActivity.photo); + if (imageInfo) { + const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId); + if (s3Url) { + structuredActivity.photo = s3Url; + } else { + logger.warn(`API call: Failed S3 upload for activity ${activityId}. Photo may be base64 or null.`); + } + } + } + structuredActivity.lastCheck = new Date().toISOString(); + await setActivityData(activityId, structuredActivity); + return { data: structuredActivity, status: 200 }; +} + +// --- API Endpoints --- +app.get('/', (req, res) => { + res.send('Welcome to the DSAS CCA API!

\ + API Endpoints:
\ + GET /v1/activity/list
\ + GET /v1/activity/:activityId (ID must be 1-4 digits)
\ + GET /v1/staffs'); +}); + +// NEW ENDPOINT: /v1/activity/list +app.get('/v1/activity/list', async (req, res) => { + try { + logger.info('Request received for /v1/activity/list'); + const activityKeys = await getAllActivityKeys(); // From redis-service.mjs + const clubList = {}; + + if (!activityKeys || activityKeys.length === 0) { + logger.info('No activity keys found in Redis for list.'); + return res.json({}); // Return empty object if no keys + } + + // Fetch all activity data in parallel + // Note: This can be many individual GETs to Redis. + // For a very large number of keys, consider if this approach is too slow. + // However, Redis is fast, and Promise.all helps manage concurrency. + const allActivityDataPromises = activityKeys.map(async (key) => { + const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); + return getActivityData(activityId); + }); + + const allActivities = await Promise.all(allActivityDataPromises); + + allActivities.forEach(activityData => { + // Check for a valid club: + // 1. activityData exists + // 2. It has an 'id' and a 'name' + // 3. It's not an error placeholder (no 'error' field) + // 4. It's not an intentionally empty record (no 'source: api-fetch-empty' field, or check if other essential fields are missing) + if (activityData && + activityData.id && + activityData.name && // Ensure name is present and not empty + !activityData.error && + activityData.source !== 'api-fetch-empty') { + clubList[activityData.id] = activityData.name; + } + }); + + logger.info(`Returning list of ${Object.keys(clubList).length} valid clubs.`); + res.json(clubList); + + } catch (error) { + logger.error('Error in /v1/activity/list endpoint:', error); + res.status(500).json({ error: 'An internal server error occurred while generating activity list.' }); + } +}); + + +app.get('/v1/activity/:activityId', async (req, res) => { + let { activityId } = req.params; + + if (!/^\d{1,4}$/.test(activityId)) { + return res.status(400).json({ error: 'Invalid Activity ID format.' }); + } + if (!USERNAME || !PASSWORD) { + logger.error('API username or password not configured.'); + return res.status(500).json({ error: 'Server configuration error.' }); + } + + try { + let cachedActivity = await getActivityData(activityId); + const isValidCacheEntry = cachedActivity && + !cachedActivity.error && + Object.keys(cachedActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length > 0; + + if (isValidCacheEntry) { + logger.info(`Cache HIT for activity ID: ${activityId}`); + cachedActivity.cache = "HIT"; + return res.json(cachedActivity); + } + + logger.info(`Cache MISS or stale/empty for activity ID: ${activityId}. Fetching...`); + const { data: liveActivity, status } = await fetchProcessAndStoreActivity(activityId); + + liveActivity.cache = "MISS"; + if (status === 404 && Object.keys(liveActivity).filter(k => k !== 'lastCheck' && k !== 'cache' && k !== 'source').length === 0) { + return res.status(404).json({ error: `Activity ${activityId} not found.`, ...liveActivity }); + } + res.status(status).json(liveActivity); + + } catch (error) { + logger.error(`Error in /v1/activity/${activityId} endpoint:`, error); + res.status(500).json({ error: 'An internal server error occurred.', cache: "ERROR" }); + } +}); + +app.get('/v1/staffs', async (req, res) => { + if (!USERNAME || !PASSWORD) { + logger.error('API username or password not configured.'); + return res.status(500).json({ error: 'Server configuration error.' }); + } + + try { + let cachedStaffs = await getStaffData(); + if (cachedStaffs && cachedStaffs.lastCheck) { + logger.info('Cache HIT for staffs.'); + cachedStaffs.cache = "HIT"; + return res.json(cachedStaffs); + } + + logger.info('Cache MISS for staffs. Fetching from source.'); + const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD); + if (activityJson) { + const staffMap = await structStaffData(activityJson); + let staffObject = Object.fromEntries(staffMap); + staffObject.lastCheck = new Date().toISOString(); + staffObject.cache = "MISS"; + await setStaffData(staffObject); + res.json(staffObject); + } else { + logger.error(`Could not retrieve base data for staffs (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`); + res.status(404).json({ error: `Could not retrieve base data for staff details.`, cache: "MISS" }); + } + } catch (error) { + logger.error('Error in /v1/staffs endpoint:', error); + res.status(500).json({ error: 'An internal server error occurred while fetching staff data.', cache: "ERROR" }); + } +}); + + +// Function to perform background initialization and periodic tasks +async function performBackgroundTasks() { + logger.info('Starting background initialization tasks...'); + try { + await initializeClubCache(); + await initializeOrUpdateStaffCache(true); + await cleanupOrphanedS3Images(); + + logger.info(`Setting up periodic club cache updates every ${CLUB_CHECK_INTERVAL_SECONDS} seconds.`); + setInterval(updateStaleClubs, CLUB_CHECK_INTERVAL_SECONDS * 1000); + + logger.info(`Setting up periodic staff cache updates every ${STAFF_CHECK_INTERVAL_SECONDS} seconds.`); + setInterval(() => initializeOrUpdateStaffCache(false), STAFF_CHECK_INTERVAL_SECONDS * 1000); + + logger.info('Background initialization and periodic task setup complete.'); + } catch (error) { + logger.error('Error during background initialization tasks:', error); + } +} + +// --- Start Server and Background Tasks --- +async function startServer() { + const redis = getRedisClient(); + if (!redis) { + logger.error('Redis client is not initialized. Server cannot start. Check REDIS_URL.'); + process.exit(1); + } + + try { + await redis.ping(); + logger.info('Redis connection confirmed.'); + + app.listen(PORT, () => { + logger.info(`Server is running on http://localhost:${PORT}`); + logger.info(`Allowed CORS origins: ${allowedOriginsEnv === '*' ? 'All (*)' : allowedOriginsEnv}`); + if (!USERNAME || !PASSWORD) { + logger.warn('Warning: API_USERNAME or API_PASSWORD is not set.'); + } + }); + + performBackgroundTasks().catch(error => { + logger.error('Unhandled error in performBackgroundTasks:', error); + }); + + } catch (err) { + logger.error('Failed to connect to Redis or critical error during server startup. Server not started.', err); + process.exit(1); + } +} + +if (process.env.NODE_ENV !== 'test') { + startServer(); +} + +process.on('SIGINT', async () => { + logger.info('Server shutting down (SIGINT)...'); + const redis = getRedisClient(); + if (redis) { + await redis.quit(); + logger.info('Redis connection closed.'); + } + process.exit(0); +}); + +process.on('SIGTERM', async () => { + logger.info('Server shutting down (SIGTERM)...'); + const redis = getRedisClient(); + if (redis) { + await redis.quit(); + logger.info('Redis connection closed.'); + } + process.exit(0); +}); + +export { app }; \ No newline at end of file diff --git a/package.json b/package.json index 5d11f16..643bd99 100644 --- a/package.json +++ b/package.json @@ -5,15 +5,19 @@ "main": "main.js", "type": "module", "scripts": { - "start": "node main.js", - "dev": "node --watch main.js" + "start": "node main.mjs", + "dev": "node --watch main.mjs" }, "dependencies": { + "@aws-sdk/client-s3": "^3.806.0", "axios": "^1.9.0", "cors": "^2.8.5", "dotenv": "^16.5.0", "express": "^5.1.0", - "node-fetch": "^3.3.2" + "ioredis": "^5.6.1", + "node-fetch": "^3.3.2", + "p-limit": "^6.2.0", + "uuid": "^11.1.0" }, "packageManager": "pnpm@10.10.0+sha512.d615db246fe70f25dcfea6d8d73dee782ce23e2245e3c4f6f888249fb568149318637dca73c2c5c8ef2a4ca0d5657fb9567188bfab47f566d1ee6ce987815c39" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f39ee75..6708b8c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.806.0 + version: 3.806.0 axios: specifier: ^1.9.0 version: 1.9.0 @@ -20,12 +23,387 @@ importers: express: specifier: ^5.1.0 version: 5.1.0 + ioredis: + specifier: ^5.6.1 + version: 5.6.1 node-fetch: specifier: ^3.3.2 version: 3.3.2 + p-limit: + specifier: ^6.2.0 + version: 6.2.0 + uuid: + specifier: ^11.1.0 + version: 11.1.0 packages: + '@aws-crypto/crc32@5.2.0': + resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/crc32c@5.2.0': + resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==} + + '@aws-crypto/sha1-browser@5.2.0': + resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==} + + '@aws-crypto/sha256-browser@5.2.0': + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + + '@aws-crypto/sha256-js@5.2.0': + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/supports-web-crypto@5.2.0': + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + + '@aws-crypto/util@5.2.0': + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + + '@aws-sdk/client-s3@3.806.0': + resolution: {integrity: sha512-kQaBBBxEBU/IJ2wKG+LL2BK+uvBwpdvOA9jy1WhW+U2/DIMwMrjVs7M/ZvTlmVOJwhZaONcJbgQqsN4Yirjj4g==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/client-sso@3.806.0': + resolution: {integrity: sha512-X0p/9/u9e6b22rlQqKucdtjdqmjSNB4c/8zDEoD5MvgYAAbMF9HNE0ST2xaA/WsJ7uE0jFfhPY2/00pslL1DqQ==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/core@3.806.0': + resolution: {integrity: sha512-HJRINPncdjPK0iL3f6cBpqCMaxVwq2oDbRCzOx04tsLZ0tNgRACBfT3d/zNVRvMt6fnOVKXoN1LAtQaw50pjEA==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/credential-provider-env@3.806.0': + resolution: {integrity: sha512-nbPwmZn0kt6Q1XI2FaJWP6AhF9tro4cO5HlmZQx8NU+B0H1y9WMo659Q5zLLY46BXgoQVIJEsPSZpcZk27O4aw==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/credential-provider-http@3.806.0': + resolution: {integrity: sha512-e/gB2iJQQ4ZpecOVpEFhEvjGwuTqNCzhVaVsFYVc49FPfR1seuN7qBGYe1MO7mouGDQFInzJgcNup0DnYUrLiw==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/credential-provider-ini@3.806.0': + resolution: {integrity: sha512-FogfbuYSEZgFxbNy0QcsBZHHe5mSv5HV3+JyB5n0kCyjOISCVCZD7gwxKdXjt8O1hXq5k5SOdQvydGULlB6rew==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/credential-provider-node@3.806.0': + resolution: {integrity: sha512-fZX8xP2Kf0k70kDTog/87fh/M+CV0E2yujSw1cUBJhDSwDX3RlUahiJk7TpB/KGw6hEFESMd6+7kq3UzYuw3rg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/credential-provider-process@3.806.0': + resolution: {integrity: sha512-8Y8GYEw/1e5IZRDQL02H6nsTDcRWid/afRMeWg+93oLQmbHcTtdm48tjis+7Xwqy+XazhMDmkbUht11QPTDJcQ==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/credential-provider-sso@3.806.0': + resolution: {integrity: sha512-hT9OBwCxWMPBydNhXm2gdNNzx5AJNheS9RglwDDvXWzQ9qDuRztjuMBilMSUMb0HF9K4IqQjYzGqczMuktz4qQ==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.806.0': + resolution: {integrity: sha512-XxaSY9Zd3D4ClUGENYMvi52ac5FuJPPAsvRtEfyrSdEpf6QufbMpnexWBZMYRF31h/VutgqtJwosGgNytpxMEg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-bucket-endpoint@3.806.0': + resolution: {integrity: sha512-ACjuyKJw9OZl8z8HzPEaqn1o7ElVW94mowyoZvyUIDouwAPGqPGJbJ5V35qx1oDTFSAJX+N3O3AO6RyFc8nUhw==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-expect-continue@3.804.0': + resolution: {integrity: sha512-YW1hySBolALMII6C8y7Z0CRG2UX1dGJjLEBNFeefhO/xP7ZuE1dvnmfJGaEuBMnvc3wkRS63VZ3aqX6sevM1CA==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-flexible-checksums@3.806.0': + resolution: {integrity: sha512-YEmuU2Nr/+blhi70gS38fnCe2IoL6OVVZXMp4MbzqZRUqeBbnxZhHQrd5YOiboJz7iq+g98xwFebHY167iejcg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-host-header@3.804.0': + resolution: {integrity: sha512-bum1hLVBrn2lJCi423Z2fMUYtsbkGI2s4N+2RI2WSjvbaVyMSv/WcejIrjkqiiMR+2Y7m5exgoKeg4/TODLDPQ==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-location-constraint@3.804.0': + resolution: {integrity: sha512-AMtKnllIWKgoo7hiJfphLYotEwTERfjVMO2+cKAncz9w1g+bnYhHxiVhJJoR94y047c06X4PU5MsTxvdQ73Znw==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-logger@3.804.0': + resolution: {integrity: sha512-w/qLwL3iq0KOPQNat0Kb7sKndl9BtceigINwBU7SpkYWX9L/Lem6f8NPEKrC9Tl4wDBht3Yztub4oRTy/horJA==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-recursion-detection@3.804.0': + resolution: {integrity: sha512-zqHOrvLRdsUdN/ehYfZ9Tf8svhbiLLz5VaWUz22YndFv6m9qaAcijkpAOlKexsv3nLBMJdSdJ6GUTAeIy3BZzw==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.806.0': + resolution: {integrity: sha512-K1ssdovHH/kPN9EUS1LznwzoL+r89Cx8qAkp0K8MqdCQuBjZ0KRnjvo9nx69Vg5d/rg01VYTxomFUPXfcPtVXw==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-ssec@3.804.0': + resolution: {integrity: sha512-Tk8jK0gOIUBvEPTz/wwSlP1V70zVQ3QYqsLPAjQRMO6zfOK9ax31dln3MgKvFDJxBydS2tS3wsn53v+brxDxTA==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/middleware-user-agent@3.806.0': + resolution: {integrity: sha512-XoIromVffgXnc+/mjlR2EVzQVIei3bPVtafIZNsHuEmUvIWJXiWsa2eJpt3BUqa0HF9YPknK7ommNEhqRb8ucg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/nested-clients@3.806.0': + resolution: {integrity: sha512-ua2gzpfQ9MF8Rny+tOAivowOWWvqEusez2rdcQK8jdBjA1ANd/0xzToSZjZh0ziN8Kl8jOhNnHbQJ0v6dT6+hg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/region-config-resolver@3.806.0': + resolution: {integrity: sha512-cuv5pX55JOlzKC/iLsB5nZ9eUyVgncim3VhhWHZA/KYPh7rLMjOEfZ+xyaE9uLJXGmzOJboFH7+YdTRdIcOgrg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.806.0': + resolution: {integrity: sha512-IrbEnpKvG8d9rUWAvsF28g8qBlQ02FaOxn4cGXtTs0b0BGMK1M+cGQrYjJ7Ak08kIXDxBqsdIlZGsKYr+Ds9+w==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/token-providers@3.806.0': + resolution: {integrity: sha512-I6SxcsvV7yinJZmPgGullFHS0tsTKa7K3jEc5dmyCz8X+kZPfsWNffZmtmnCvWXPqMXWBvK6hVaxwomx79yeHA==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/types@3.804.0': + resolution: {integrity: sha512-A9qnsy9zQ8G89vrPPlNG9d1d8QcKRGqJKqwyGgS0dclJpwy6d1EWgQLIolKPl6vcFpLoe6avLOLxr+h8ur5wpg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/util-arn-parser@3.804.0': + resolution: {integrity: sha512-wmBJqn1DRXnZu3b4EkE6CWnoWMo1ZMvlfkqU5zPz67xx1GMaXlDCchFvKAXMjk4jn/L1O3tKnoFDNsoLV1kgNQ==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/util-endpoints@3.806.0': + resolution: {integrity: sha512-3YRRgZ+qFuWDdm5uAbxKsr65UAil4KkrFKua9f4m7Be3v24ETiFOOqhanFUIk9/WOtvzF7oFEiDjYKDGlwV2xg==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/util-locate-window@3.804.0': + resolution: {integrity: sha512-zVoRfpmBVPodYlnMjgVjfGoEZagyRF5IPn3Uo6ZvOZp24chnW/FRstH7ESDHDDRga4z3V+ElUQHKpFDXWyBW5A==} + engines: {node: '>=18.0.0'} + + '@aws-sdk/util-user-agent-browser@3.804.0': + resolution: {integrity: sha512-KfW6T6nQHHM/vZBBdGn6fMyG/MgX5lq82TDdX4HRQRRuHKLgBWGpKXqqvBwqIaCdXwWHgDrg2VQups6GqOWW2A==} + + '@aws-sdk/util-user-agent-node@3.806.0': + resolution: {integrity: sha512-Az2e4/gmPZ4BpB7QRj7U76I+fctXhNcxlcgsaHnMhvt+R30nvzM2EhsyBUvsWl8+r9bnLeYt9BpvEZeq2ANDzA==} + engines: {node: '>=18.0.0'} + peerDependencies: + aws-crt: '>=1.0.0' + peerDependenciesMeta: + aws-crt: + optional: true + + '@aws-sdk/xml-builder@3.804.0': + resolution: {integrity: sha512-JbGWp36IG9dgxtvC6+YXwt5WDZYfuamWFtVfK6fQpnmL96dx+GUPOXPKRWdw67WLKf2comHY28iX2d3z35I53Q==} + engines: {node: '>=18.0.0'} + + '@ioredis/commands@1.2.0': + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + + '@smithy/abort-controller@4.0.2': + resolution: {integrity: sha512-Sl/78VDtgqKxN2+1qduaVE140XF+Xg+TafkncspwM4jFP/LHr76ZHmIY/y3V1M0mMLNk+Je6IGbzxy23RSToMw==} + engines: {node: '>=18.0.0'} + + '@smithy/chunked-blob-reader-native@4.0.0': + resolution: {integrity: sha512-R9wM2yPmfEMsUmlMlIgSzOyICs0x9uu7UTHoccMyt7BWw8shcGM8HqB355+BZCPBcySvbTYMs62EgEQkNxz2ig==} + engines: {node: '>=18.0.0'} + + '@smithy/chunked-blob-reader@5.0.0': + resolution: {integrity: sha512-+sKqDBQqb036hh4NPaUiEkYFkTUGYzRsn3EuFhyfQfMy6oGHEUJDurLP9Ufb5dasr/XiAmPNMr6wa9afjQB+Gw==} + engines: {node: '>=18.0.0'} + + '@smithy/config-resolver@4.1.2': + resolution: {integrity: sha512-7r6mZGwb5LmLJ+zPtkLoznf2EtwEuSWdtid10pjGl/7HefCE4mueOkrfki8JCUm99W6UfP47/r3tbxx9CfBN5A==} + engines: {node: '>=18.0.0'} + + '@smithy/core@3.3.1': + resolution: {integrity: sha512-W7AppgQD3fP1aBmo8wWo0id5zeR2/aYRy067vZsDVaa6v/mdhkg6DxXwEVuSPjZl+ZnvWAQbUMCd5ckw38+tHQ==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.0.4': + resolution: {integrity: sha512-jN6M6zaGVyB8FmNGG+xOPQB4N89M1x97MMdMnm1ESjljLS3Qju/IegQizKujaNcy2vXAvrz0en8bobe6E55FEA==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-codec@4.0.2': + resolution: {integrity: sha512-p+f2kLSK7ZrXVfskU/f5dzksKTewZk8pJLPvER3aFHPt76C2MxD9vNatSfLzzQSQB4FNO96RK4PSXfhD1TTeMQ==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-browser@4.0.2': + resolution: {integrity: sha512-CepZCDs2xgVUtH7ZZ7oDdZFH8e6Y2zOv8iiX6RhndH69nlojCALSKK+OXwZUgOtUZEUaZ5e1hULVCHYbCn7pug==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-config-resolver@4.1.0': + resolution: {integrity: sha512-1PI+WPZ5TWXrfj3CIoKyUycYynYJgZjuQo8U+sphneOtjsgrttYybdqESFReQrdWJ+LKt6NEdbYzmmfDBmjX2A==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-node@4.0.2': + resolution: {integrity: sha512-C5bJ/C6x9ENPMx2cFOirspnF9ZsBVnBMtP6BdPl/qYSuUawdGQ34Lq0dMcf42QTjUZgWGbUIZnz6+zLxJlb9aw==} + engines: {node: '>=18.0.0'} + + '@smithy/eventstream-serde-universal@4.0.2': + resolution: {integrity: sha512-St8h9JqzvnbB52FtckiHPN4U/cnXcarMniXRXTKn0r4b4XesZOGiAyUdj1aXbqqn1icSqBlzzUsCl6nPB018ng==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.0.2': + resolution: {integrity: sha512-+9Dz8sakS9pe7f2cBocpJXdeVjMopUDLgZs1yWeu7h++WqSbjUYv/JAJwKwXw1HV6gq1jyWjxuyn24E2GhoEcQ==} + engines: {node: '>=18.0.0'} + + '@smithy/hash-blob-browser@4.0.2': + resolution: {integrity: sha512-3g188Z3DyhtzfBRxpZjU8R9PpOQuYsbNnyStc/ZVS+9nVX1f6XeNOa9IrAh35HwwIZg+XWk8bFVtNINVscBP+g==} + engines: {node: '>=18.0.0'} + + '@smithy/hash-node@4.0.2': + resolution: {integrity: sha512-VnTpYPnRUE7yVhWozFdlxcYknv9UN7CeOqSrMH+V877v4oqtVYuoqhIhtSjmGPvYrYnAkaM61sLMKHvxL138yg==} + engines: {node: '>=18.0.0'} + + '@smithy/hash-stream-node@4.0.2': + resolution: {integrity: sha512-POWDuTznzbIwlEXEvvXoPMS10y0WKXK790soe57tFRfvf4zBHyzE529HpZMqmDdwG9MfFflnyzndUQ8j78ZdSg==} + engines: {node: '>=18.0.0'} + + '@smithy/invalid-dependency@4.0.2': + resolution: {integrity: sha512-GatB4+2DTpgWPday+mnUkoumP54u/MDM/5u44KF9hIu8jF0uafZtQLcdfIKkIcUNuF/fBojpLEHZS/56JqPeXQ==} + engines: {node: '>=18.0.0'} + + '@smithy/is-array-buffer@2.2.0': + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + + '@smithy/is-array-buffer@4.0.0': + resolution: {integrity: sha512-saYhF8ZZNoJDTvJBEWgeBccCg+yvp1CX+ed12yORU3NilJScfc6gfch2oVb4QgxZrGUx3/ZJlb+c/dJbyupxlw==} + engines: {node: '>=18.0.0'} + + '@smithy/md5-js@4.0.2': + resolution: {integrity: sha512-Hc0R8EiuVunUewCse2syVgA2AfSRco3LyAv07B/zCOMa+jpXI9ll+Q21Nc6FAlYPcpNcAXqBzMhNs1CD/pP2bA==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-content-length@4.0.2': + resolution: {integrity: sha512-hAfEXm1zU+ELvucxqQ7I8SszwQ4znWMbNv6PLMndN83JJN41EPuS93AIyh2N+gJ6x8QFhzSO6b7q2e6oClDI8A==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-endpoint@4.1.4': + resolution: {integrity: sha512-qWyYvszzvDjT2AxRvEpNhnMTo8QX9MCAtuSA//kYbXewb+2mEGQCk1UL4dNIrKLcF5KT11dOJtxFYT0kzajq5g==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-retry@4.1.5': + resolution: {integrity: sha512-eQguCTA2TRGyg4P7gDuhRjL2HtN5OKJXysq3Ufj0EppZe4XBmSyKIvVX9ws9KkD3lkJskw1tfE96wMFsiUShaw==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-serde@4.0.3': + resolution: {integrity: sha512-rfgDVrgLEVMmMn0BI8O+8OVr6vXzjV7HZj57l0QxslhzbvVfikZbVfBVthjLHqib4BW44QhcIgJpvebHlRaC9A==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-stack@4.0.2': + resolution: {integrity: sha512-eSPVcuJJGVYrFYu2hEq8g8WWdJav3sdrI4o2c6z/rjnYDd3xH9j9E7deZQCzFn4QvGPouLngH3dQ+QVTxv5bOQ==} + engines: {node: '>=18.0.0'} + + '@smithy/node-config-provider@4.1.1': + resolution: {integrity: sha512-1slS5jf5icHETwl5hxEVBj+mh6B+LbVW4yRINsGtUKH+nxM5Pw2H59+qf+JqYFCHp9jssG4vX81f5WKnjMN3Vw==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.0.4': + resolution: {integrity: sha512-/mdqabuAT3o/ihBGjL94PUbTSPSRJ0eeVTdgADzow0wRJ0rN4A27EOrtlK56MYiO1fDvlO3jVTCxQtQmK9dZ1g==} + engines: {node: '>=18.0.0'} + + '@smithy/property-provider@4.0.2': + resolution: {integrity: sha512-wNRoQC1uISOuNc2s4hkOYwYllmiyrvVXWMtq+TysNRVQaHm4yoafYQyjN/goYZS+QbYlPIbb/QRjaUZMuzwQ7A==} + engines: {node: '>=18.0.0'} + + '@smithy/protocol-http@5.1.0': + resolution: {integrity: sha512-KxAOL1nUNw2JTYrtviRRjEnykIDhxc84qMBzxvu1MUfQfHTuBlCG7PA6EdVwqpJjH7glw7FqQoFxUJSyBQgu7g==} + engines: {node: '>=18.0.0'} + + '@smithy/querystring-builder@4.0.2': + resolution: {integrity: sha512-NTOs0FwHw1vimmQM4ebh+wFQvOwkEf/kQL6bSM1Lock+Bv4I89B3hGYoUEPkmvYPkDKyp5UdXJYu+PoTQ3T31Q==} + engines: {node: '>=18.0.0'} + + '@smithy/querystring-parser@4.0.2': + resolution: {integrity: sha512-v6w8wnmZcVXjfVLjxw8qF7OwESD9wnpjp0Dqry/Pod0/5vcEA3qxCr+BhbOHlxS8O+29eLpT3aagxXGwIoEk7Q==} + engines: {node: '>=18.0.0'} + + '@smithy/service-error-classification@4.0.3': + resolution: {integrity: sha512-FTbcajmltovWMjj3tksDQdD23b2w6gH+A0DYA1Yz3iSpjDj8fmkwy62UnXcWMy4d5YoMoSyLFHMfkEVEzbiN8Q==} + engines: {node: '>=18.0.0'} + + '@smithy/shared-ini-file-loader@4.0.2': + resolution: {integrity: sha512-J9/gTWBGVuFZ01oVA6vdb4DAjf1XbDhK6sLsu3OS9qmLrS6KB5ygpeHiM3miIbj1qgSJ96GYszXFWv6ErJ8QEw==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.1.0': + resolution: {integrity: sha512-4t5WX60sL3zGJF/CtZsUQTs3UrZEDO2P7pEaElrekbLqkWPYkgqNW1oeiNYC6xXifBnT9dVBOnNQRvOE9riU9w==} + engines: {node: '>=18.0.0'} + + '@smithy/smithy-client@4.2.4': + resolution: {integrity: sha512-oolSEpr/ABUtVmFMdNgi6sSXsK4csV9n4XM9yXgvDJGRa32tQDUdv9s+ztFZKccay1AiTWLSGsyDj2xy1gsv7Q==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.2.0': + resolution: {integrity: sha512-7eMk09zQKCO+E/ivsjQv+fDlOupcFUCSC/L2YUPgwhvowVGWbPQHjEFcmjt7QQ4ra5lyowS92SV53Zc6XD4+fg==} + engines: {node: '>=18.0.0'} + + '@smithy/url-parser@4.0.2': + resolution: {integrity: sha512-Bm8n3j2ScqnT+kJaClSVCMeiSenK6jVAzZCNewsYWuZtnBehEz4r2qP0riZySZVfzB+03XZHJeqfmJDkeeSLiQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-base64@4.0.0': + resolution: {integrity: sha512-CvHfCmO2mchox9kjrtzoHkWHxjHZzaFojLc8quxXY7WAAMAg43nuxwv95tATVgQFNDwd4M9S1qFzj40Ul41Kmg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-body-length-browser@4.0.0': + resolution: {integrity: sha512-sNi3DL0/k64/LO3A256M+m3CDdG6V7WKWHdAiBBMUN8S3hK3aMPhwnPik2A/a2ONN+9doY9UxaLfgqsIRg69QA==} + engines: {node: '>=18.0.0'} + + '@smithy/util-body-length-node@4.0.0': + resolution: {integrity: sha512-q0iDP3VsZzqJyje8xJWEJCNIu3lktUGVoSy1KB0UWym2CL1siV3artm+u1DFYTLejpsrdGyCSWBdGNjJzfDPjg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-buffer-from@2.2.0': + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + + '@smithy/util-buffer-from@4.0.0': + resolution: {integrity: sha512-9TOQ7781sZvddgO8nxueKi3+yGvkY35kotA0Y6BWRajAv8jjmigQ1sBwz0UX47pQMYXJPahSKEKYFgt+rXdcug==} + engines: {node: '>=18.0.0'} + + '@smithy/util-config-provider@4.0.0': + resolution: {integrity: sha512-L1RBVzLyfE8OXH+1hsJ8p+acNUSirQnWQ6/EgpchV88G6zGBTDPdXiiExei6Z1wR2RxYvxY/XLw6AMNCCt8H3w==} + engines: {node: '>=18.0.0'} + + '@smithy/util-defaults-mode-browser@4.0.12': + resolution: {integrity: sha512-0vPKiC+rXWMq397tsa/RFcO/kJ1UsibgNCXScMsRwzm9WMT4QjGf43zVPWZ5hPLu3z/1XddiZFIlKcu2j/yUuQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-defaults-mode-node@4.0.12': + resolution: {integrity: sha512-zCx9noceM3Pw2jvcJ3w3RbvKnPe3lCo6txH9ksZj6CeRZPkvRZPLXmKVSOvDr9QQP3VRq/WnBLd+LTZAL7+0IQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-endpoints@3.0.4': + resolution: {integrity: sha512-VfFATC1bmZLV2858B/O1NpMcL32wYo8DPPhHxYxDCodDl3f3mSZ5oJheW1IF91A0EeAADz2WsakM/hGGPGNKLg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-hex-encoding@4.0.0': + resolution: {integrity: sha512-Yk5mLhHtfIgW2W2WQZWSg5kuMZCVbvhFmC7rV4IO2QqnZdbEFPmQnCcGMAX2z/8Qj3B9hYYNjZOhWym+RwhePw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-middleware@4.0.2': + resolution: {integrity: sha512-6GDamTGLuBQVAEuQ4yDQ+ti/YINf/MEmIegrEeg7DdB/sld8BX1lqt9RRuIcABOhAGTA50bRbPzErez7SlDtDQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-retry@4.0.3': + resolution: {integrity: sha512-DPuYjZQDXmKr/sNvy9Spu8R/ESa2e22wXZzSAY6NkjOLj6spbIje/Aq8rT97iUMdDj0qHMRIe+bTxvlU74d9Ng==} + engines: {node: '>=18.0.0'} + + '@smithy/util-stream@4.2.0': + resolution: {integrity: sha512-Vj1TtwWnuWqdgQI6YTUF5hQ/0jmFiOYsc51CSMgj7QfyO+RF4EnT2HNjoviNlOOmgzgvf3f5yno+EiC4vrnaWQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-uri-escape@4.0.0': + resolution: {integrity: sha512-77yfbCbQMtgtTylO9itEAdpPXSog3ZxMe09AEhm0dU0NLTalV70ghDZFR+Nfi1C60jnJoh/Re4090/DuZh2Omg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-utf8@2.3.0': + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@4.0.0': + resolution: {integrity: sha512-b+zebfKCfRdgNJDknHCob3O7FpeYQN6ZG6YLExMcasDHsCXlsXCEuiPZeLnJLpwa5dvPetGlnGCiMHuLwGvFow==} + engines: {node: '>=18.0.0'} + + '@smithy/util-waiter@4.0.3': + resolution: {integrity: sha512-JtaY3FxmD+te+KSI2FJuEcfNC9T/DGGVf551babM7fAaXhjJUt7oSYurH1Devxd2+BOSUACCgt3buinx4UnmEA==} + engines: {node: '>=18.0.0'} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -40,6 +418,9 @@ packages: resolution: {integrity: sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==} engines: {node: '>=18'} + bowser@2.11.0: + resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -52,6 +433,10 @@ packages: resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} engines: {node: '>= 0.4'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -93,6 +478,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -139,6 +528,10 @@ packages: resolution: {integrity: sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==} engines: {node: '>= 18'} + fast-xml-parser@4.4.1: + resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==} + hasBin: true + fetch-blob@3.2.0: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} @@ -210,6 +603,10 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ioredis@5.6.1: + resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==} + engines: {node: '>=12.22.0'} + ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -217,6 +614,12 @@ packages: is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -276,6 +679,10 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + p-limit@6.2.0: + resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} + engines: {node: '>=18'} + parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -303,6 +710,14 @@ packages: resolution: {integrity: sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==} engines: {node: '>= 0.8'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + router@2.2.0: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} @@ -340,14 +755,23 @@ packages: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} + strnum@1.1.2: + resolution: {integrity: sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==} + toidentifier@1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + type-is@2.0.1: resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} engines: {node: '>= 0.6'} @@ -356,6 +780,14 @@ packages: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + hasBin: true + vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} @@ -367,8 +799,803 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + yocto-queue@1.2.1: + resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==} + engines: {node: '>=12.20'} + snapshots: + '@aws-crypto/crc32@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.804.0 + tslib: 2.8.1 + + '@aws-crypto/crc32c@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.804.0 + tslib: 2.8.1 + + '@aws-crypto/sha1-browser@5.2.0': + dependencies: + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-locate-window': 3.804.0 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-browser@5.2.0': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-locate-window': 3.804.0 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-js@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.804.0 + tslib: 2.8.1 + + '@aws-crypto/supports-web-crypto@5.2.0': + dependencies: + tslib: 2.8.1 + + '@aws-crypto/util@5.2.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.806.0': + dependencies: + '@aws-crypto/sha1-browser': 5.2.0 + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.806.0 + '@aws-sdk/credential-provider-node': 3.806.0 + '@aws-sdk/middleware-bucket-endpoint': 3.806.0 + '@aws-sdk/middleware-expect-continue': 3.804.0 + '@aws-sdk/middleware-flexible-checksums': 3.806.0 + '@aws-sdk/middleware-host-header': 3.804.0 + '@aws-sdk/middleware-location-constraint': 3.804.0 + '@aws-sdk/middleware-logger': 3.804.0 + '@aws-sdk/middleware-recursion-detection': 3.804.0 + '@aws-sdk/middleware-sdk-s3': 3.806.0 + '@aws-sdk/middleware-ssec': 3.804.0 + '@aws-sdk/middleware-user-agent': 3.806.0 + '@aws-sdk/region-config-resolver': 3.806.0 + '@aws-sdk/signature-v4-multi-region': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-endpoints': 3.806.0 + '@aws-sdk/util-user-agent-browser': 3.804.0 + '@aws-sdk/util-user-agent-node': 3.806.0 + '@aws-sdk/xml-builder': 3.804.0 + '@smithy/config-resolver': 4.1.2 + '@smithy/core': 3.3.1 + '@smithy/eventstream-serde-browser': 4.0.2 + '@smithy/eventstream-serde-config-resolver': 4.1.0 + '@smithy/eventstream-serde-node': 4.0.2 + '@smithy/fetch-http-handler': 5.0.2 + '@smithy/hash-blob-browser': 4.0.2 + '@smithy/hash-node': 4.0.2 + '@smithy/hash-stream-node': 4.0.2 + '@smithy/invalid-dependency': 4.0.2 + '@smithy/md5-js': 4.0.2 + '@smithy/middleware-content-length': 4.0.2 + '@smithy/middleware-endpoint': 4.1.4 + '@smithy/middleware-retry': 4.1.5 + '@smithy/middleware-serde': 4.0.3 + '@smithy/middleware-stack': 4.0.2 + '@smithy/node-config-provider': 4.1.1 + '@smithy/node-http-handler': 4.0.4 + '@smithy/protocol-http': 5.1.0 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + '@smithy/url-parser': 4.0.2 + '@smithy/util-base64': 4.0.0 + '@smithy/util-body-length-browser': 4.0.0 + '@smithy/util-body-length-node': 4.0.0 + '@smithy/util-defaults-mode-browser': 4.0.12 + '@smithy/util-defaults-mode-node': 4.0.12 + '@smithy/util-endpoints': 3.0.4 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-retry': 4.0.3 + '@smithy/util-stream': 4.2.0 + '@smithy/util-utf8': 4.0.0 + '@smithy/util-waiter': 4.0.3 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/client-sso@3.806.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.806.0 + '@aws-sdk/middleware-host-header': 3.804.0 + '@aws-sdk/middleware-logger': 3.804.0 + '@aws-sdk/middleware-recursion-detection': 3.804.0 + '@aws-sdk/middleware-user-agent': 3.806.0 + '@aws-sdk/region-config-resolver': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-endpoints': 3.806.0 + '@aws-sdk/util-user-agent-browser': 3.804.0 + '@aws-sdk/util-user-agent-node': 3.806.0 + '@smithy/config-resolver': 4.1.2 + '@smithy/core': 3.3.1 + '@smithy/fetch-http-handler': 5.0.2 + '@smithy/hash-node': 4.0.2 + '@smithy/invalid-dependency': 4.0.2 + '@smithy/middleware-content-length': 4.0.2 + '@smithy/middleware-endpoint': 4.1.4 + '@smithy/middleware-retry': 4.1.5 + '@smithy/middleware-serde': 4.0.3 + '@smithy/middleware-stack': 4.0.2 + '@smithy/node-config-provider': 4.1.1 + '@smithy/node-http-handler': 4.0.4 + '@smithy/protocol-http': 5.1.0 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + '@smithy/url-parser': 4.0.2 + '@smithy/util-base64': 4.0.0 + '@smithy/util-body-length-browser': 4.0.0 + '@smithy/util-body-length-node': 4.0.0 + '@smithy/util-defaults-mode-browser': 4.0.12 + '@smithy/util-defaults-mode-node': 4.0.12 + '@smithy/util-endpoints': 3.0.4 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-retry': 4.0.3 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/core@3.806.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/core': 3.3.1 + '@smithy/node-config-provider': 4.1.1 + '@smithy/property-provider': 4.0.2 + '@smithy/protocol-http': 5.1.0 + '@smithy/signature-v4': 5.1.0 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + '@smithy/util-middleware': 4.0.2 + fast-xml-parser: 4.4.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.806.0': + dependencies: + '@aws-sdk/core': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/property-provider': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.806.0': + dependencies: + '@aws-sdk/core': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/fetch-http-handler': 5.0.2 + '@smithy/node-http-handler': 4.0.4 + '@smithy/property-provider': 4.0.2 + '@smithy/protocol-http': 5.1.0 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + '@smithy/util-stream': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.806.0': + dependencies: + '@aws-sdk/core': 3.806.0 + '@aws-sdk/credential-provider-env': 3.806.0 + '@aws-sdk/credential-provider-http': 3.806.0 + '@aws-sdk/credential-provider-process': 3.806.0 + '@aws-sdk/credential-provider-sso': 3.806.0 + '@aws-sdk/credential-provider-web-identity': 3.806.0 + '@aws-sdk/nested-clients': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/credential-provider-imds': 4.0.4 + '@smithy/property-provider': 4.0.2 + '@smithy/shared-ini-file-loader': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-node@3.806.0': + dependencies: + '@aws-sdk/credential-provider-env': 3.806.0 + '@aws-sdk/credential-provider-http': 3.806.0 + '@aws-sdk/credential-provider-ini': 3.806.0 + '@aws-sdk/credential-provider-process': 3.806.0 + '@aws-sdk/credential-provider-sso': 3.806.0 + '@aws-sdk/credential-provider-web-identity': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/credential-provider-imds': 4.0.4 + '@smithy/property-provider': 4.0.2 + '@smithy/shared-ini-file-loader': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-process@3.806.0': + dependencies: + '@aws-sdk/core': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/property-provider': 4.0.2 + '@smithy/shared-ini-file-loader': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.806.0': + dependencies: + '@aws-sdk/client-sso': 3.806.0 + '@aws-sdk/core': 3.806.0 + '@aws-sdk/token-providers': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/property-provider': 4.0.2 + '@smithy/shared-ini-file-loader': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/credential-provider-web-identity@3.806.0': + dependencies: + '@aws-sdk/core': 3.806.0 + '@aws-sdk/nested-clients': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/property-provider': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/middleware-bucket-endpoint@3.806.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-arn-parser': 3.804.0 + '@smithy/node-config-provider': 4.1.1 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + '@smithy/util-config-provider': 4.0.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-expect-continue@3.804.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-flexible-checksums@3.806.0': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@aws-crypto/crc32c': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/core': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/is-array-buffer': 4.0.0 + '@smithy/node-config-provider': 4.1.1 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-stream': 4.2.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-host-header@3.804.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-location-constraint@3.804.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-logger@3.804.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-recursion-detection@3.804.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.806.0': + dependencies: + '@aws-sdk/core': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-arn-parser': 3.804.0 + '@smithy/core': 3.3.1 + '@smithy/node-config-provider': 4.1.1 + '@smithy/protocol-http': 5.1.0 + '@smithy/signature-v4': 5.1.0 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + '@smithy/util-config-provider': 4.0.0 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-stream': 4.2.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-ssec@3.804.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-user-agent@3.806.0': + dependencies: + '@aws-sdk/core': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-endpoints': 3.806.0 + '@smithy/core': 3.3.1 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.806.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.806.0 + '@aws-sdk/middleware-host-header': 3.804.0 + '@aws-sdk/middleware-logger': 3.804.0 + '@aws-sdk/middleware-recursion-detection': 3.804.0 + '@aws-sdk/middleware-user-agent': 3.806.0 + '@aws-sdk/region-config-resolver': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@aws-sdk/util-endpoints': 3.806.0 + '@aws-sdk/util-user-agent-browser': 3.804.0 + '@aws-sdk/util-user-agent-node': 3.806.0 + '@smithy/config-resolver': 4.1.2 + '@smithy/core': 3.3.1 + '@smithy/fetch-http-handler': 5.0.2 + '@smithy/hash-node': 4.0.2 + '@smithy/invalid-dependency': 4.0.2 + '@smithy/middleware-content-length': 4.0.2 + '@smithy/middleware-endpoint': 4.1.4 + '@smithy/middleware-retry': 4.1.5 + '@smithy/middleware-serde': 4.0.3 + '@smithy/middleware-stack': 4.0.2 + '@smithy/node-config-provider': 4.1.1 + '@smithy/node-http-handler': 4.0.4 + '@smithy/protocol-http': 5.1.0 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + '@smithy/url-parser': 4.0.2 + '@smithy/util-base64': 4.0.0 + '@smithy/util-body-length-browser': 4.0.0 + '@smithy/util-body-length-node': 4.0.0 + '@smithy/util-defaults-mode-browser': 4.0.12 + '@smithy/util-defaults-mode-node': 4.0.12 + '@smithy/util-endpoints': 3.0.4 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-retry': 4.0.3 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/region-config-resolver@3.806.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/node-config-provider': 4.1.1 + '@smithy/types': 4.2.0 + '@smithy/util-config-provider': 4.0.0 + '@smithy/util-middleware': 4.0.2 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.806.0': + dependencies: + '@aws-sdk/middleware-sdk-s3': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/protocol-http': 5.1.0 + '@smithy/signature-v4': 5.1.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.806.0': + dependencies: + '@aws-sdk/nested-clients': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/property-provider': 4.0.2 + '@smithy/shared-ini-file-loader': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + + '@aws-sdk/types@3.804.0': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/util-arn-parser@3.804.0': + dependencies: + tslib: 2.8.1 + + '@aws-sdk/util-endpoints@3.806.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/types': 4.2.0 + '@smithy/util-endpoints': 3.0.4 + tslib: 2.8.1 + + '@aws-sdk/util-locate-window@3.804.0': + dependencies: + tslib: 2.8.1 + + '@aws-sdk/util-user-agent-browser@3.804.0': + dependencies: + '@aws-sdk/types': 3.804.0 + '@smithy/types': 4.2.0 + bowser: 2.11.0 + tslib: 2.8.1 + + '@aws-sdk/util-user-agent-node@3.806.0': + dependencies: + '@aws-sdk/middleware-user-agent': 3.806.0 + '@aws-sdk/types': 3.804.0 + '@smithy/node-config-provider': 4.1.1 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.804.0': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@ioredis/commands@1.2.0': {} + + '@smithy/abort-controller@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/chunked-blob-reader-native@4.0.0': + dependencies: + '@smithy/util-base64': 4.0.0 + tslib: 2.8.1 + + '@smithy/chunked-blob-reader@5.0.0': + dependencies: + tslib: 2.8.1 + + '@smithy/config-resolver@4.1.2': + dependencies: + '@smithy/node-config-provider': 4.1.1 + '@smithy/types': 4.2.0 + '@smithy/util-config-provider': 4.0.0 + '@smithy/util-middleware': 4.0.2 + tslib: 2.8.1 + + '@smithy/core@3.3.1': + dependencies: + '@smithy/middleware-serde': 4.0.3 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + '@smithy/util-body-length-browser': 4.0.0 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-stream': 4.2.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.0.4': + dependencies: + '@smithy/node-config-provider': 4.1.1 + '@smithy/property-provider': 4.0.2 + '@smithy/types': 4.2.0 + '@smithy/url-parser': 4.0.2 + tslib: 2.8.1 + + '@smithy/eventstream-codec@4.0.2': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.2.0 + '@smithy/util-hex-encoding': 4.0.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-browser@4.0.2': + dependencies: + '@smithy/eventstream-serde-universal': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-config-resolver@4.1.0': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-node@4.0.2': + dependencies: + '@smithy/eventstream-serde-universal': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/eventstream-serde-universal@4.0.2': + dependencies: + '@smithy/eventstream-codec': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.0.2': + dependencies: + '@smithy/protocol-http': 5.1.0 + '@smithy/querystring-builder': 4.0.2 + '@smithy/types': 4.2.0 + '@smithy/util-base64': 4.0.0 + tslib: 2.8.1 + + '@smithy/hash-blob-browser@4.0.2': + dependencies: + '@smithy/chunked-blob-reader': 5.0.0 + '@smithy/chunked-blob-reader-native': 4.0.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/hash-node@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + '@smithy/util-buffer-from': 4.0.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@smithy/hash-stream-node@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@smithy/invalid-dependency@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/is-array-buffer@2.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/is-array-buffer@4.0.0': + dependencies: + tslib: 2.8.1 + + '@smithy/md5-js@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@smithy/middleware-content-length@4.0.2': + dependencies: + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/middleware-endpoint@4.1.4': + dependencies: + '@smithy/core': 3.3.1 + '@smithy/middleware-serde': 4.0.3 + '@smithy/node-config-provider': 4.1.1 + '@smithy/shared-ini-file-loader': 4.0.2 + '@smithy/types': 4.2.0 + '@smithy/url-parser': 4.0.2 + '@smithy/util-middleware': 4.0.2 + tslib: 2.8.1 + + '@smithy/middleware-retry@4.1.5': + dependencies: + '@smithy/node-config-provider': 4.1.1 + '@smithy/protocol-http': 5.1.0 + '@smithy/service-error-classification': 4.0.3 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-retry': 4.0.3 + tslib: 2.8.1 + uuid: 9.0.1 + + '@smithy/middleware-serde@4.0.3': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/middleware-stack@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/node-config-provider@4.1.1': + dependencies: + '@smithy/property-provider': 4.0.2 + '@smithy/shared-ini-file-loader': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.0.4': + dependencies: + '@smithy/abort-controller': 4.0.2 + '@smithy/protocol-http': 5.1.0 + '@smithy/querystring-builder': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/property-provider@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/protocol-http@5.1.0': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/querystring-builder@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + '@smithy/util-uri-escape': 4.0.0 + tslib: 2.8.1 + + '@smithy/querystring-parser@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/service-error-classification@4.0.3': + dependencies: + '@smithy/types': 4.2.0 + + '@smithy/shared-ini-file-loader@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/signature-v4@5.1.0': + dependencies: + '@smithy/is-array-buffer': 4.0.0 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + '@smithy/util-hex-encoding': 4.0.0 + '@smithy/util-middleware': 4.0.2 + '@smithy/util-uri-escape': 4.0.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@smithy/smithy-client@4.2.4': + dependencies: + '@smithy/core': 3.3.1 + '@smithy/middleware-endpoint': 4.1.4 + '@smithy/middleware-stack': 4.0.2 + '@smithy/protocol-http': 5.1.0 + '@smithy/types': 4.2.0 + '@smithy/util-stream': 4.2.0 + tslib: 2.8.1 + + '@smithy/types@4.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/url-parser@4.0.2': + dependencies: + '@smithy/querystring-parser': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-base64@4.0.0': + dependencies: + '@smithy/util-buffer-from': 4.0.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@smithy/util-body-length-browser@4.0.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-body-length-node@4.0.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-buffer-from@2.2.0': + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-buffer-from@4.0.0': + dependencies: + '@smithy/is-array-buffer': 4.0.0 + tslib: 2.8.1 + + '@smithy/util-config-provider@4.0.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-defaults-mode-browser@4.0.12': + dependencies: + '@smithy/property-provider': 4.0.2 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + bowser: 2.11.0 + tslib: 2.8.1 + + '@smithy/util-defaults-mode-node@4.0.12': + dependencies: + '@smithy/config-resolver': 4.1.2 + '@smithy/credential-provider-imds': 4.0.4 + '@smithy/node-config-provider': 4.1.1 + '@smithy/property-provider': 4.0.2 + '@smithy/smithy-client': 4.2.4 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-endpoints@3.0.4': + dependencies: + '@smithy/node-config-provider': 4.1.1 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-hex-encoding@4.0.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-middleware@4.0.2': + dependencies: + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-retry@4.0.3': + dependencies: + '@smithy/service-error-classification': 4.0.3 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + + '@smithy/util-stream@4.2.0': + dependencies: + '@smithy/fetch-http-handler': 5.0.2 + '@smithy/node-http-handler': 4.0.4 + '@smithy/types': 4.2.0 + '@smithy/util-base64': 4.0.0 + '@smithy/util-buffer-from': 4.0.0 + '@smithy/util-hex-encoding': 4.0.0 + '@smithy/util-utf8': 4.0.0 + tslib: 2.8.1 + + '@smithy/util-uri-escape@4.0.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-utf8@2.3.0': + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-utf8@4.0.0': + dependencies: + '@smithy/util-buffer-from': 4.0.0 + tslib: 2.8.1 + + '@smithy/util-waiter@4.0.3': + dependencies: + '@smithy/abort-controller': 4.0.2 + '@smithy/types': 4.2.0 + tslib: 2.8.1 + accepts@2.0.0: dependencies: mime-types: 3.0.1 @@ -398,6 +1625,8 @@ snapshots: transitivePeerDependencies: - supports-color + bowser@2.11.0: {} + bytes@3.1.2: {} call-bind-apply-helpers@1.0.2: @@ -410,6 +1639,8 @@ snapshots: call-bind-apply-helpers: 1.0.2 get-intrinsic: 1.3.0 + cluster-key-slot@1.1.2: {} + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -437,6 +1668,8 @@ snapshots: delayed-stream@1.0.0: {} + denque@2.1.0: {} + depd@2.0.0: {} dotenv@16.5.0: {} @@ -502,6 +1735,10 @@ snapshots: transitivePeerDependencies: - supports-color + fast-xml-parser@4.4.1: + dependencies: + strnum: 1.1.2 + fetch-blob@3.2.0: dependencies: node-domexception: 1.0.0 @@ -581,10 +1818,28 @@ snapshots: inherits@2.0.4: {} + ioredis@5.6.1: + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.4.0 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ipaddr.js@1.9.1: {} is-promise@4.0.0: {} + lodash.defaults@4.2.0: {} + + lodash.isarguments@3.1.0: {} + math-intrinsics@1.1.0: {} media-typer@1.1.0: {} @@ -627,6 +1882,10 @@ snapshots: dependencies: wrappy: 1.0.2 + p-limit@6.2.0: + dependencies: + yocto-queue: 1.2.1 + parseurl@1.3.3: {} path-to-regexp@8.2.0: {} @@ -651,6 +1910,12 @@ snapshots: iconv-lite: 0.6.3 unpipe: 1.0.0 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + router@2.2.0: dependencies: debug: 4.4.0 @@ -720,10 +1985,16 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + standard-as-callback@2.1.0: {} + statuses@2.0.1: {} + strnum@1.1.2: {} + toidentifier@1.0.1: {} + tslib@2.8.1: {} + type-is@2.0.1: dependencies: content-type: 1.0.5 @@ -732,8 +2003,14 @@ snapshots: unpipe@1.0.0: {} + uuid@11.1.0: {} + + uuid@9.0.1: {} + vary@1.1.2: {} web-streams-polyfill@3.3.3: {} wrappy@1.0.2: {} + + yocto-queue@1.2.1: {} diff --git a/services/cache-manager.mjs b/services/cache-manager.mjs new file mode 100644 index 0000000..1d96216 --- /dev/null +++ b/services/cache-manager.mjs @@ -0,0 +1,209 @@ +// services/cache-manager.mjs +import dotenv from 'dotenv'; +import pLimit from 'p-limit'; +import { fetchActivityData } from '../engage-api/get-activity.mjs'; +import { structActivityData } from '../engage-api/struct-activity.mjs'; +import { structStaffData } from '../engage-api/struct-staff.mjs'; +import { + getActivityData, + setActivityData, + getStaffData, + setStaffData, + getAllActivityKeys as getAllRedisActivityKeys, // Renamed import for clarity + ACTIVITY_KEY_PREFIX +} from './redis-service.mjs'; +import { uploadImageFromBase64, listS3Objects, deleteS3Objects, constructS3Url } from './s3-service.mjs'; +import { extractBase64Image } from '../utils/image-processor.mjs'; +import { logger } from '../utils/logger.mjs'; + +dotenv.config(); + +const USERNAME = process.env.API_USERNAME; +const PASSWORD = process.env.API_PASSWORD; +const MAX_ACTIVITY_ID_SCAN = parseInt(process.env.MAX_ACTIVITY_ID_SCAN || '9999', 10); +const CONCURRENT_API_CALLS = parseInt(process.env.CONCURRENT_API_CALLS || '10', 10); +const CLUB_UPDATE_INTERVAL_MINS = parseInt(process.env.CLUB_UPDATE_INTERVAL_MINS || '60', 10); +const STAFF_UPDATE_INTERVAL_MINS = parseInt(process.env.STAFF_UPDATE_INTERVAL_MINS || '60', 10); +const FIXED_STAFF_ACTIVITY_ID = process.env.FIXED_STAFF_ACTIVITY_ID; +const S3_IMAGE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').replace(/\/$/, ''); // Ensure no trailing slash + +const limit = pLimit(CONCURRENT_API_CALLS); + +async function processAndCacheActivity(activityId) { + logger.debug(`Processing activity ID: ${activityId}`); + try { + const activityJson = await fetchActivityData(activityId, USERNAME, PASSWORD); + let structuredActivity; + + if (!activityJson) { + logger.info(`No data found for activity ID ${activityId} from engage API. Caching as empty.`); + structuredActivity = {}; + } else { + structuredActivity = await structActivityData(activityJson); + if (structuredActivity && structuredActivity.photo && structuredActivity.photo.startsWith('data:image')) { + const imageInfo = extractBase64Image(structuredActivity.photo); + if (imageInfo) { + const s3Url = await uploadImageFromBase64(imageInfo.base64Content, imageInfo.format, activityId); + if (s3Url) { + structuredActivity.photo = s3Url; + } else { + logger.warn(`Failed to upload image to S3 for activity ${activityId}. Base64 photo data will be removed or kept as is depending on structActivityData's behavior if upload fails.`); + // Potentially set photo to null or remove if upload is critical and fails + // structuredActivity.photo = null; // Example + } + } + } + } + + structuredActivity.lastCheck = new Date().toISOString(); + await setActivityData(activityId, structuredActivity); + // logger.info(`Successfully processed and cached activity ID: ${activityId}`); // Can be too verbose + return structuredActivity; + } catch (error) { + logger.error(`Error processing activity ID ${activityId}:`, error); + const errorData = { lastCheck: new Date().toISOString(), error: "Failed to fetch or process" }; + await setActivityData(activityId, errorData); + return errorData; + } +} + +export async function initializeClubCache() { + logger.info(`Starting initial club cache population up to ID ${MAX_ACTIVITY_ID_SCAN}...`); + const promises = []; + for (let i = 0; i <= MAX_ACTIVITY_ID_SCAN; i++) { + const activityId = String(i); + promises.push(limit(async () => { + const cachedData = await getActivityData(activityId); + if (!cachedData || Object.keys(cachedData).length === 0 || !cachedData.lastCheck || cachedData.error) { + logger.debug(`Initializing cache for activity ID: ${activityId}`); + await processAndCacheActivity(activityId); + } + })); + } + await Promise.all(promises); + logger.info('Initial club cache population finished.'); +} + +export async function updateStaleClubs() { + logger.info('Starting stale club check...'); + const now = Date.now(); + const updateIntervalMs = CLUB_UPDATE_INTERVAL_MINS * 60 * 1000; + const promises = []; + const activityKeys = await getAllRedisActivityKeys(); // More efficient than iterating 0-MAX_ID if many are empty + + for (const key of activityKeys) { + const activityId = key.substring(ACTIVITY_KEY_PREFIX.length); + promises.push(limit(async () => { + const cachedData = await getActivityData(activityId); // Re-fetch to get latest before deciding + if (cachedData && cachedData.lastCheck) { + const lastCheckTime = new Date(cachedData.lastCheck).getTime(); + if ((now - lastCheckTime) > updateIntervalMs || cachedData.error) { // Also update if previous fetch had an error + logger.info(`Activity ${activityId} is stale or had error. Updating...`); + await processAndCacheActivity(activityId); + } + } else if (!cachedData || Object.keys(cachedData).length === 0) { + // This case handles if a key was deleted or somehow became totally empty + logger.info(`Activity ${activityId} not in cache or is empty object. Attempting to fetch...`); + await processAndCacheActivity(activityId); + } + })); + } + // Optionally, iterate 0-MAX_ID_SCAN for any IDs not yet in Redis (newly created activities) + // This part can be heavy. Consider if getAllRedisActivityKeys is sufficient for "staleness". + // For truly new activities, they'd be picked up on direct API call or a less frequent full scan. + + await Promise.all(promises); + logger.info('Stale club check finished.'); +} + + +export async function initializeOrUpdateStaffCache(forceUpdate = false) { + logger.info('Starting staff cache check/update...'); + try { + const cachedStaffData = await getStaffData(); + const now = Date.now(); + const updateIntervalMs = STAFF_UPDATE_INTERVAL_MINS * 60 * 1000; + let needsUpdate = forceUpdate; + + if (!cachedStaffData || !cachedStaffData.lastCheck) { + needsUpdate = true; + } else { + const lastCheckTime = new Date(cachedStaffData.lastCheck).getTime(); + if ((now - lastCheckTime) > updateIntervalMs) { + needsUpdate = true; + } + } + + if (needsUpdate) { + logger.info('Staff data needs update. Fetching...'); + const activityJson = await fetchActivityData(FIXED_STAFF_ACTIVITY_ID, USERNAME, PASSWORD); + if (activityJson) { + const staffMap = await structStaffData(activityJson); + const staffObject = Object.fromEntries(staffMap); + staffObject.lastCheck = new Date().toISOString(); + await setStaffData(staffObject); + logger.info('Staff data updated and cached.'); + } else { + logger.warn(`Could not retrieve base data for staff (activity ID ${FIXED_STAFF_ACTIVITY_ID}).`); + if(cachedStaffData && cachedStaffData.lastCheck){ // If old data exists, just update its lastCheck + cachedStaffData.lastCheck = new Date().toISOString(); + await setStaffData(cachedStaffData); + } + } + } else { + logger.info('Staff data is up-to-date.'); + } + } catch (error) { + logger.error('Error initializing or updating staff cache:', error); + } +} + +export async function cleanupOrphanedS3Images() { + logger.info('Starting S3 orphan image cleanup...'); + const s3ObjectListPrefix = S3_IMAGE_PREFIX ? `${S3_IMAGE_PREFIX}/` : ''; // Ensure trailing slash for prefix listing + + try { + const referencedS3Urls = new Set(); + const allActivityRedisKeys = await getAllRedisActivityKeys(); + + for (const redisKey of allActivityRedisKeys) { + const activityId = redisKey.substring(ACTIVITY_KEY_PREFIX.length); + const activityData = await getActivityData(activityId); // Fetch by ID + if (activityData && typeof activityData.photo === 'string' && activityData.photo.startsWith('http')) { // Assuming S3 URLs start with http/https + // Check if the photo URL matches the expected S3 endpoint structure + if (activityData.photo.startsWith(process.env.S3_ENDPOINT)) { + referencedS3Urls.add(activityData.photo); + } + } + } + logger.info(`Found ${referencedS3Urls.size} unique S3 URLs referenced in Redis.`); + + const s3ObjectKeys = await listS3Objects(s3ObjectListPrefix); + if (!s3ObjectKeys || s3ObjectKeys.length === 0) { + logger.info(`No images found in S3 under prefix "${s3ObjectListPrefix}". Nothing to clean up.`); + return; + } + logger.debug(`Found ${s3ObjectKeys.length} objects in S3 under prefix "${s3ObjectListPrefix}".`); + + const orphanedObjectKeys = []; + for (const objectKey of s3ObjectKeys) { + const s3Url = constructS3Url(objectKey); // Construct URL from key to compare + if (!referencedS3Urls.has(s3Url)) { + orphanedObjectKeys.push(objectKey); + } + } + + if (orphanedObjectKeys.length > 0) { + logger.info(`Found ${orphanedObjectKeys.length} orphaned S3 objects to delete. Submitting deletion...`); + // orphanedObjectKeys.forEach(key => logger.debug(`Orphaned key: ${key}`)); // For verbose debugging + await deleteS3Objects(orphanedObjectKeys); + } else { + logger.info('No orphaned S3 images found after comparison.'); + } + + logger.info('S3 orphan image cleanup finished.'); + + } catch (error) { + logger.error('Error during S3 orphan image cleanup:', error); + } +} \ No newline at end of file diff --git a/services/redis-service.mjs b/services/redis-service.mjs new file mode 100644 index 0000000..527789c --- /dev/null +++ b/services/redis-service.mjs @@ -0,0 +1,135 @@ +// services/redis-service.mjs +import Redis from 'ioredis'; +import dotenv from 'dotenv'; +import { logger } from '../utils/logger.mjs'; + +dotenv.config(); + +const redisUrl = process.env.REDIS_URL; +let redisClient; + +export const ACTIVITY_KEY_PREFIX = 'activity:'; // Exported for use in cache-manager +const STAFF_KEY = 'staffs:all'; + +try { + if (redisUrl) { + redisClient = new Redis(redisUrl); + + redisClient.on('connect', () => { + logger.info('Connected to Redis successfully!'); + }); + + redisClient.on('error', (err) => { + logger.error('Redis connection error:', err); + }); + } else { + logger.error('REDIS_URL not defined. Redis client not initialized.'); + } +} catch (error) { + logger.error('Failed to initialize Redis client:', error); +} + + +/** + * Gets activity data from Redis. + * @param {string} activityId + * @returns {Promise} Parsed JSON object or null if not found/error. + */ +export async function getActivityData(activityId) { + if (!redisClient) { + logger.warn('Redis client not available, skipping getActivityData'); + return null; + } + try { + const data = await redisClient.get(`${ACTIVITY_KEY_PREFIX}${activityId}`); + return data ? JSON.parse(data) : null; + } catch (err) { + logger.error(`Error getting activity ${activityId} from Redis:`, err); + return null; + } +} + +/** + * Sets activity data in Redis. + * @param {string} activityId + * @param {object} data The activity data object. + * @returns {Promise} + */ +export async function setActivityData(activityId, data) { + if (!redisClient) { + logger.warn('Redis client not available, skipping setActivityData'); + return; + } + try { + await redisClient.set(`${ACTIVITY_KEY_PREFIX}${activityId}`, JSON.stringify(data)); + } catch (err) { + logger.error(`Error setting activity ${activityId} in Redis:`, err); + } +} + +/** + * Gets staff data from Redis. + * @returns {Promise} Parsed JSON object or null if not found/error. + */ +export async function getStaffData() { + if (!redisClient) { + logger.warn('Redis client not available, skipping getStaffData'); + return null; + } + try { + const data = await redisClient.get(STAFF_KEY); + return data ? JSON.parse(data) : null; + } catch (err) { + logger.error('Error getting staff data from Redis:', err); + return null; + } +} + +/** + * Sets staff data in Redis. + * @param {object} data The staff data object. + * @returns {Promise} + */ +export async function setStaffData(data) { + if (!redisClient) { + logger.warn('Redis client not available, skipping setStaffData'); + return; + } + try { + await redisClient.set(STAFF_KEY, JSON.stringify(data)); + } catch (err) { + logger.error('Error setting staff data in Redis:', err); + } +} + +/** + * Gets all activity keys from Redis. + * This can be resource-intensive on large datasets. Use with caution. + * @returns {Promise} An array of keys. + */ +export async function getAllActivityKeys() { + if (!redisClient) { + logger.warn('Redis client not available, skipping getAllActivityKeys'); + return []; + } + try { + // Using SCAN for better performance than KEYS on production Redis + let cursor = '0'; + const keys = []; + do { + const [nextCursor, foundKeys] = await redisClient.scan(cursor, 'MATCH', `${ACTIVITY_KEY_PREFIX}*`, 'COUNT', '100'); + keys.push(...foundKeys); + cursor = nextCursor; + } while (cursor !== '0'); + logger.info(`Found ${keys.length} activity keys in Redis using SCAN.`); + return keys; + } catch (err) { + logger.error('Error getting all activity keys from Redis using SCAN:', err); + return []; // Fallback or indicate error + } +} + + +export function getRedisClient() { + return redisClient; +} \ No newline at end of file diff --git a/services/s3-service.mjs b/services/s3-service.mjs new file mode 100644 index 0000000..adf9c12 --- /dev/null +++ b/services/s3-service.mjs @@ -0,0 +1,186 @@ +// services/s3-service.mjs +import { S3Client, PutObjectCommand, ListObjectsV2Command, DeleteObjectsCommand } from '@aws-sdk/client-s3'; +import { v4 as uuidv4 } from 'uuid'; +import dotenv from 'dotenv'; +import { logger } from '../utils/logger.mjs'; +import { decodeBase64Image } from '../utils/image-processor.mjs'; + +dotenv.config(); + +const S3_ENDPOINT = process.env.S3_ENDPOINT; +const S3_REGION = process.env.S3_REGION; +const S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID; +const S3_SECRET_ACCESS_KEY = process.env.S3_SECRET_ACCESS_KEY; +const BUCKET_NAME = process.env.S3_BUCKET_NAME; +const PUBLIC_URL_FILE_PREFIX = (process.env.S3_PUBLIC_URL_PREFIX || 'files').replace(/\/$/, ''); // Ensures no trailing slash + +let s3Client; + +if (S3_ENDPOINT && S3_REGION && S3_ACCESS_KEY_ID && S3_SECRET_ACCESS_KEY && BUCKET_NAME) { + s3Client = new S3Client({ + endpoint: S3_ENDPOINT, + region: S3_REGION, + credentials: { + accessKeyId: S3_ACCESS_KEY_ID, + secretAccessKey: S3_SECRET_ACCESS_KEY, + }, + forcePathStyle: true, // Important for MinIO and some S3-compatibles + }); +} else { + logger.warn('S3 client configuration is incomplete. S3 operations will be disabled.'); +} + + +/** + * Uploads an image from a base64 string to S3. + * @param {string} base64Data The base64 content (without the data URI prefix). + * @param {string} originalFormat The image format (e.g., 'png', 'jpeg'). + * @param {string} activityId The activity ID, used for naming. + * @returns {Promise} The public URL of the uploaded image or null on error. + */ +export async function uploadImageFromBase64(base64Data, originalFormat, activityId) { + if (!s3Client) { + logger.warn('S3 client not configured. Cannot upload image.'); + return null; + } + if (!base64Data || !originalFormat || !activityId) { + logger.error('S3 Upload: Missing base64Data, originalFormat, or activityId'); + return null; + } + + try { + const imageBuffer = decodeBase64Image(base64Data); + // Ensure PUBLIC_URL_FILE_PREFIX is part of the key + const objectKey = `${PUBLIC_URL_FILE_PREFIX}/activity-${activityId}-${uuidv4()}.${originalFormat}`; + + const params = { + Bucket: BUCKET_NAME, + Key: objectKey, + Body: imageBuffer, + ContentType: `image/${originalFormat}`, + ACL: 'public-read', + }; + + await s3Client.send(new PutObjectCommand(params)); + const publicUrl = constructS3Url(objectKey); + logger.info(`Image uploaded to S3: ${publicUrl}`); + return publicUrl; + } catch (error) { + logger.error(`S3 Upload Error for activity ${activityId}:`, error); + return null; + } +} + +/** + * Lists all objects in the S3 bucket under a specific prefix. + * @param {string} prefix The prefix to filter objects by (e.g., S3_PUBLIC_URL_PREFIX + '/'). + * @returns {Promise>} A list of object keys. + */ +export async function listS3Objects(prefix) { + if (!s3Client) { + logger.warn('S3 client not configured. Cannot list objects.'); + return []; + } + const objectKeys = []; + let isTruncated = true; + let continuationToken; + + logger.debug(`Listing objects from S3 with prefix: "${prefix}"`); + const listCommandInput = { // Renamed to avoid conflict if command is redefined in loop + Bucket: BUCKET_NAME, + Prefix: prefix, + }; + + try { + while (isTruncated) { + if (continuationToken) { + listCommandInput.ContinuationToken = continuationToken; + } + const command = new ListObjectsV2Command(listCommandInput); + const { Contents, IsTruncated: NextIsTruncated, NextContinuationToken } = await s3Client.send(command); + + if (Contents) { + Contents.forEach(item => { + if (item.Key && !item.Key.endsWith('/')) { // Ensure it's a file, not a pseudo-directory + objectKeys.push(item.Key); + } + }); + } + isTruncated = NextIsTruncated; + continuationToken = NextContinuationToken; + } + logger.info(`Listed ${objectKeys.length} object keys from S3 with prefix "${prefix}"`); + return objectKeys; + } catch (error) { + logger.error(`S3 ListObjects Error with prefix "${prefix}":`, error); + return []; + } +} + + +/** + * Deletes multiple objects from S3. + * @param {Array} objectKeysArray Array of object keys to delete. + * @returns {Promise} True if successful or partially successful, false on major error. + */ +export async function deleteS3Objects(objectKeysArray) { + if (!s3Client) { + logger.warn('S3 client not configured. Cannot delete objects.'); + return false; + } + if (!objectKeysArray || objectKeysArray.length === 0) { + logger.info('No objects to delete from S3.'); + return true; + } + + const MAX_DELETE_COUNT = 1000; // S3 API limit + let allDeletionsSuccessful = true; + + for (let i = 0; i < objectKeysArray.length; i += MAX_DELETE_COUNT) { + const chunk = objectKeysArray.slice(i, i + MAX_DELETE_COUNT); + const deleteParams = { + Bucket: BUCKET_NAME, + Delete: { + Objects: chunk.map(key => ({ Key: key })), + Quiet: false, // We want error details + }, + }; + try { + const command = new DeleteObjectsCommand(deleteParams); + const output = await s3Client.send(command); + if (output.Errors && output.Errors.length > 0) { + allDeletionsSuccessful = false; + output.Errors.forEach(err => { + logger.error(`S3 Delete Error for key ${err.Key}: ${err.Message}`); + }); + } + if (output.Deleted && output.Deleted.length > 0) { + logger.info(`Successfully submitted deletion for ${output.Deleted.length} objects from S3 chunk (some might have failed, check individual errors).`); + } + } catch (error) { + logger.error('S3 DeleteObjects Command Error for a chunk:', error); + allDeletionsSuccessful = false; + } + } + if (allDeletionsSuccessful && objectKeysArray.length > 0) { + logger.info(`Finished S3 deletion request for ${objectKeysArray.length} keys.`); + } else if (objectKeysArray.length > 0) { + logger.warn(`S3 deletion request for ${objectKeysArray.length} keys completed with some errors.`); + } + return allDeletionsSuccessful; +} + +/** + * Constructs the public S3 URL for an object key. + * @param {string} objectKey The key of the object in S3. + * @returns {string} The full public URL. + */ +export function constructS3Url(objectKey) { + // Ensure S3_ENDPOINT does not end with a slash + const s3Base = S3_ENDPOINT.replace(/\/$/, ''); + // Ensure BUCKET_NAME does not start or end with a slash + const bucket = BUCKET_NAME.replace(/^\//, '').replace(/\/$/, ''); + // Ensure objectKey does not start with a slash + const key = objectKey.replace(/^\//, ''); + return `${s3Base}/${bucket}/${key}`; +} \ No newline at end of file diff --git a/utils/image-processor.mjs b/utils/image-processor.mjs new file mode 100644 index 0000000..55a51c4 --- /dev/null +++ b/utils/image-processor.mjs @@ -0,0 +1,43 @@ +// utils/image-processor.mjs +import { logger } from './logger.mjs'; // Using the logger + +/** + * Extracts base64 content and format from a data URL string. + * E.g., "data:image/jpeg;base64,xxxxxxxxxxxxxxx" + * @param {string} dataUrl The full data URL string. + * @returns {object|null} An object { base64Content: string, format: string } or null if not found. + */ +export function extractBase64Image(dataUrl) { + if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) { + return null; + } + + const markers = [ + { prefix: "data:image/png;base64,", format: "png" }, + { prefix: "data:image/jpeg;base64,", format: "jpeg" }, + { prefix: "data:image/jpg;base64,", format: "jpg" }, + { prefix: "data:image/gif;base64,", format: "gif" }, + { prefix: "data:image/svg+xml;base64,", format: "svg" }, // svg+xml -> svg + { prefix: "data:image/webp;base64,", format: "webp" } + ]; + + for (const marker of markers) { + if (dataUrl.startsWith(marker.prefix)) { + const base64Content = dataUrl.substring(marker.prefix.length); + logger.debug(`Found image of format: ${marker.format}`); + return { base64Content, format: marker.format }; + } + } + + logger.warn("No known base64 image marker found in the provided data URL:", dataUrl.substring(0, 50) + "..."); + return null; +} + +/** + * Decodes a base64 string to a Buffer. + * @param {string} base64String The base64 encoded string (without the data URI prefix). + * @returns {Buffer} + */ +export function decodeBase64Image(base64String) { + return Buffer.from(base64String, 'base64'); +} \ No newline at end of file diff --git a/utils/logger.mjs b/utils/logger.mjs new file mode 100644 index 0000000..9b44d13 --- /dev/null +++ b/utils/logger.mjs @@ -0,0 +1,30 @@ +// utils/logger.mjs +import dotenv from 'dotenv'; +dotenv.config(); // Ensure .env variables are loaded + +const LOG_LEVEL = process.env.LOG_LEVEL || 'info'; // Example: 'debug', 'info', 'warn', 'error' + +const levels = { + error: 0, + warn: 1, + info: 2, + debug: 3, +}; + +const currentLevel = levels[LOG_LEVEL.toLowerCase()] ?? levels.info; + +const log = (level, ...args) => { + if (levels[level] <= currentLevel) { + const timestamp = new Date().toISOString(); + console[level](`[${timestamp}] [${level.toUpperCase()}]`, ...args); + } +}; + +export const logger = { + error: (...args) => log('error', ...args), + warn: (...args) => log('warn', ...args), + info: (...args) => log('info', ...args), + debug: (...args) => log('debug', ...args), +}; + +export default logger; \ No newline at end of file