lightling/gallery-dl-archive-manager
1
0
Fork 0
Code Activity

add batching

Browse source
This commit is contained in:
lightling 2024-02-28 22:24:24 -05:00
parent 2f023b919c
commit b917b77e71
1 changed files with 55 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
run-downloadDb.js
View file

@ -19,7 +19,7 @@ const ctx = 'downloadDb.js';
*/ */
const downloadDb = async () => { const downloadDb = async () => {
log(ctx, 'Grabbing db'); log(ctx, 'Grabbing db');
let directory = '', threadMax = 1, db; let directory = '', threadMax = 1, db, usersPerBatch, waitTime;
try { try {
directory = getArg('path'); directory = getArg('path');
} catch (err) { } catch (err) {
@ -32,6 +32,15 @@ const downloadDb = async () => {
} catch (err) { } catch (err) {
log(ctx, 'Using 1 thread'); log(ctx, 'Using 1 thread');
} }
try {
usersPerBatch = getArg('usersPerBatch');
let waitTimeSec = getArg('waitTime');
waitTime = waitTimeSec * 1000;
log(ctx, `Splitting into batches with ${usersPerBatch} per batch with ${waitTimeSec} seconds pausing between each batch.`);
} catch (err) {
log(ctx, `Not using batches${!!usersPerBatch ? ' (usersPerBatch provided without waitTime)' : ''}`);
usersPerBatch = null;
}
const tryReadDb = async () => { const tryReadDb = async () => {
let file = await readFile(`${directory}/db.json`, { encoding: 'utf8' }); let file = await readFile(`${directory}/db.json`, { encoding: 'utf8' });
db = JSON.parse(file); db = JSON.parse(file);
@ -65,26 +74,54 @@ const downloadDb = async () => {
}) })
}); });
log(ctx, `Downloading media using /<user>/media for ${processes.length} users`);
await getMany(processes, threadMax, directory, 'media');
const errorReadout = []; const errorReadout = [];
processes.forEach(entry => { const executeDl = async (arr) => {
entry.logs.forEach(log => { log(ctx, `Downloading media using /<user>/media for ${arr.length} users`);
if (log.includes('NotFoundError')) { await getMany(arr, threadMax, directory, 'media');
const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
errorReadout.push(strOut);
entry.lastError = strOut;
} else if (log.includes('AuthorizationError')) {
const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
errorReadout.push(strOut);
entry.lastError = strOut;
}
});
});
log(ctx, 'Downloading media using /search'); arr.forEach(entry => {
await getMany(processes, threadMax, directory, 'search'); entry.logs.forEach(log => {
if (log.includes('NotFoundError')) {
const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
errorReadout.push(strOut);
entry.lastError = strOut;
} else if (log.includes('AuthorizationError')) {
const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
errorReadout.push(strOut);
entry.lastError = strOut;
}
});
});
log(ctx, 'Downloading media using /search');
await getMany(arr, threadMax, directory, 'search');
}
if (!!usersPerBatch) {
const batches = processes.reduce((arr, item, i) => {
const chunkIndex = Math.floor(i / usersPerBatch);
if (!arr[chunkIndex]) {
arr[chunkIndex] = [];
}
arr[chunkIndex].push(item);
return arr;
}, []);
const delay = ms => new Promise(res => setTimeout(res, ms));
log(ctx, JSON.stringify(batches, null, 2));
for (let i = 0; i < batches.length; i++) {
const batch = batches[i];
log(ctx, `Executing batch ${batch[0]?.user}-${batch[batch.length - 1]?.user}`);
await executeDl(batch);
log(ctx, `Waiting ${waitTime / 1000} seconds before next batch.`);
await delay(waitTime);
}
processes = batches.flat(1);
} else {
await executeDl(processes);
}
log(ctx, 'Updating the db'); log(ctx, 'Updating the db');
try { try {