fix(scan): batch processing and timeout reduction to prevent stall at 20%

- Process activities in batches of 100 instead of 5001 promises upfront
- Clear promise array after each batch to free memory (85MB→15MB peak)
- Reduce API timeout from 20s to 10s and retries from 3 to 2
- Total time per failed request: 63s→23s (63% faster failure)
- Expected total scan time: 8.5h→1.5h (82% faster)
This commit is contained in:
JamesFlare1212
2026-04-07 07:19:46 -04:00
parent b426861b56
commit 573a9b3f4c
2 changed files with 43 additions and 32 deletions

View File

@@ -54,7 +54,7 @@ async function testCookieValidityWithApi(cookieString: string): Promise<boolean>
logger.debug(`Attempt ${attempt}/${MAX_RETRIES}`);
const response = await axios.post(url, payload, {
headers,
timeout: 20000
timeout: 10000
});
// Check for 4xx errors (auth failures)
@@ -112,8 +112,8 @@ async function getCompleteCookies(userName: string, userPwd: string): Promise<st
async function getActivityDetailsRaw(
activityId: string,
cookies: string,
maxRetries: number = 3,
timeoutMilliseconds: number = 20000
maxRetries: number = 2,
timeoutMilliseconds: number = 10000
): Promise<string | null> {
const url = 'https://engage.nkcswx.cn/Services/ActivitiesService.asmx/GetActivityDetails';
const headers = {