Lightling
3933505fc6
- rate limit is stderr, so need to check to make sure only finishing on other errors - color errors in log - log errors to userDb - use fallback date in case dates weren't logged (e.g. /media failed) - report on not-found users, which may indicate a username change or deleted account - report on authorization errors
60 lines
1.7 KiB
JavaScript
60 lines
1.7 KiB
JavaScript
import { readFile } from 'fs/promises';
|
|
|
|
import { getArg } from './lib/args.js';
|
|
import { getMany } from './lib/dl.js';
|
|
import { error, log } from './lib/log.js';
|
|
|
|
const ctx = 'downloadDb.js';
|
|
|
|
/**
|
|
* Downloads all media possible for the users stored in db.json at the specified `--path`.
|
|
* Useful for first run or for augmenting existing media
|
|
* if it may be only partially archived in an uncertain state.
|
|
*/
|
|
const downloadDb = async () => {
|
|
log(ctx, 'Grabbing db');
|
|
let directory = '', threadMax = 1, db;
|
|
try {
|
|
directory = getArg('path');
|
|
} catch (err) {
|
|
error(ctx, err);
|
|
return;
|
|
}
|
|
try {
|
|
threadMax = getArg('threads');
|
|
log(ctx, `Using ${threadMax} threads`);
|
|
} catch (err) {
|
|
log(ctx, 'Using 1 thread');
|
|
}
|
|
try {
|
|
let file = await readFile(`${directory}/db.json`, { encoding: 'utf8' });
|
|
db = JSON.parse(file);
|
|
} catch (err) {
|
|
error(ctx, err);
|
|
return;
|
|
}
|
|
|
|
let processes = db.map(entry => ({
|
|
...entry,
|
|
logs: [],
|
|
}));
|
|
|
|
log(ctx, `Downloading media using <user>/media for ${processes.length} users`);
|
|
await getMany(processes, threadMax, directory, 'media');
|
|
|
|
log(ctx, 'Downloading media using /search');
|
|
await getMany(processes, threadMax, directory, 'search');
|
|
|
|
processes.forEach(entry => {
|
|
entry.logs.forEach(log => {
|
|
if (log.includes('NotFoundError')) {
|
|
error(ctx, `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`);
|
|
} else if (log.includes('AuthorizationError')) {
|
|
error(ctx, `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`);
|
|
}
|
|
});
|
|
});
|
|
log(ctx, 'Done');
|
|
}
|
|
|
|
downloadDb();
|