add batching
This commit is contained in:
parent
2f023b919c
commit
b917b77e71
1 changed files with 55 additions and 18 deletions
|
@ -19,7 +19,7 @@ const ctx = 'downloadDb.js';
|
|||
*/
|
||||
const downloadDb = async () => {
|
||||
log(ctx, 'Grabbing db');
|
||||
let directory = '', threadMax = 1, db;
|
||||
let directory = '', threadMax = 1, db, usersPerBatch, waitTime;
|
||||
try {
|
||||
directory = getArg('path');
|
||||
} catch (err) {
|
||||
|
@ -32,6 +32,15 @@ const downloadDb = async () => {
|
|||
} catch (err) {
|
||||
log(ctx, 'Using 1 thread');
|
||||
}
|
||||
try {
|
||||
usersPerBatch = getArg('usersPerBatch');
|
||||
let waitTimeSec = getArg('waitTime');
|
||||
waitTime = waitTimeSec * 1000;
|
||||
log(ctx, `Splitting into batches with ${usersPerBatch} per batch with ${waitTimeSec} seconds pausing between each batch.`);
|
||||
} catch (err) {
|
||||
log(ctx, `Not using batches${!!usersPerBatch ? ' (usersPerBatch provided without waitTime)' : ''}`);
|
||||
usersPerBatch = null;
|
||||
}
|
||||
const tryReadDb = async () => {
|
||||
let file = await readFile(`${directory}/db.json`, { encoding: 'utf8' });
|
||||
db = JSON.parse(file);
|
||||
|
@ -65,26 +74,54 @@ const downloadDb = async () => {
|
|||
})
|
||||
});
|
||||
|
||||
log(ctx, `Downloading media using /<user>/media for ${processes.length} users`);
|
||||
await getMany(processes, threadMax, directory, 'media');
|
||||
|
||||
const errorReadout = [];
|
||||
processes.forEach(entry => {
|
||||
entry.logs.forEach(log => {
|
||||
if (log.includes('NotFoundError')) {
|
||||
const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
|
||||
errorReadout.push(strOut);
|
||||
entry.lastError = strOut;
|
||||
} else if (log.includes('AuthorizationError')) {
|
||||
const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
|
||||
errorReadout.push(strOut);
|
||||
entry.lastError = strOut;
|
||||
}
|
||||
});
|
||||
});
|
||||
const executeDl = async (arr) => {
|
||||
log(ctx, `Downloading media using /<user>/media for ${arr.length} users`);
|
||||
await getMany(arr, threadMax, directory, 'media');
|
||||
|
||||
log(ctx, 'Downloading media using /search');
|
||||
await getMany(processes, threadMax, directory, 'search');
|
||||
arr.forEach(entry => {
|
||||
entry.logs.forEach(log => {
|
||||
if (log.includes('NotFoundError')) {
|
||||
const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
|
||||
errorReadout.push(strOut);
|
||||
entry.lastError = strOut;
|
||||
} else if (log.includes('AuthorizationError')) {
|
||||
const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
|
||||
errorReadout.push(strOut);
|
||||
entry.lastError = strOut;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
log(ctx, 'Downloading media using /search');
|
||||
await getMany(arr, threadMax, directory, 'search');
|
||||
}
|
||||
|
||||
if (!!usersPerBatch) {
|
||||
const batches = processes.reduce((arr, item, i) => {
|
||||
const chunkIndex = Math.floor(i / usersPerBatch);
|
||||
if (!arr[chunkIndex]) {
|
||||
arr[chunkIndex] = [];
|
||||
}
|
||||
arr[chunkIndex].push(item);
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
const delay = ms => new Promise(res => setTimeout(res, ms));
|
||||
log(ctx, JSON.stringify(batches, null, 2));
|
||||
|
||||
for (let i = 0; i < batches.length; i++) {
|
||||
const batch = batches[i];
|
||||
log(ctx, `Executing batch ${batch[0]?.user}-${batch[batch.length - 1]?.user}`);
|
||||
await executeDl(batch);
|
||||
log(ctx, `Waiting ${waitTime / 1000} seconds before next batch.`);
|
||||
await delay(waitTime);
|
||||
}
|
||||
|
||||
processes = batches.flat(1);
|
||||
} else {
|
||||
await executeDl(processes);
|
||||
}
|
||||
|
||||
log(ctx, 'Updating the db');
|
||||
try {
|
||||
|
|
Loading…
Add table
Reference in a new issue