113 lines
3 KiB
JavaScript
113 lines
3 KiB
JavaScript
import { readFile, writeFile } from 'fs/promises';
|
|
|
|
import { getArg, validateSiteArg, SITES } from './lib/args.js';
|
|
import { getMany } from './lib/dl.js';
|
|
import { error, log } from './lib/log.js';
|
|
import { createDb, userSchema } from './lib/schema.js';
|
|
|
|
const ctx = 'downloadUser.js';
|
|
|
|
/**
|
|
* Downloads all media possible for comma-separated `--users` stored/to-be-stored in db.json at the specified `--path`.
|
|
* Useful for first run or for augmenting existing media
|
|
* if it may be only partially archived in an uncertain state.
|
|
* Safely checks if the db.json doesn't yet exist and/or the user(s) isn't stored in the db.json at the directory provided.
|
|
*/
|
|
const downloadUsers = async () => {
|
|
log(ctx, 'Grabbing db');
|
|
let directory, threadMax = 1, users, db, site;
|
|
try {
|
|
directory = getArg('path');
|
|
users = getArg('users').split(',');
|
|
site = validateSiteArg(getArg('site'));
|
|
} catch (err) {
|
|
error(ctx, err);
|
|
return;
|
|
}
|
|
try {
|
|
threadMax = getArg('threads');
|
|
log(ctx, `Using ${threadMax} threads`);
|
|
} catch (err) {
|
|
log(ctx, 'Using 1 thread');
|
|
}
|
|
const tryReadDb = async () => {
|
|
let file = await readFile(`${directory}/db.json`, { encoding: 'utf8' });
|
|
db = JSON.parse(file);
|
|
}
|
|
try {
|
|
await tryReadDb();
|
|
} catch (err) {
|
|
if (err.toString().includes('ENOENT')) {
|
|
try {
|
|
log(ctx, 'Database was not yet present. Creating it now.');
|
|
await createDb(directory);
|
|
await tryReadDb();
|
|
} catch (err2) {
|
|
error(ctx, err2);
|
|
return;
|
|
}
|
|
} else {
|
|
error(ctx, err);
|
|
return;
|
|
}
|
|
}
|
|
|
|
let processes = users.map(user => ({
|
|
...db.userList[user] | {},
|
|
user,
|
|
lastUpdated: Date.now(),
|
|
logs: [],
|
|
}));
|
|
|
|
log(ctx, `Downloading media using /<user>/media for ${processes.length} users`);
|
|
await getMany({
|
|
userDb: processes,
|
|
threadMax,
|
|
directory,
|
|
mode: 'media',
|
|
site,
|
|
});
|
|
|
|
if (site === SITES.TWITTER) {
|
|
log(ctx, 'Downloading media using /search');
|
|
await getMany({
|
|
userDb: processes,
|
|
threadMax,
|
|
directory,
|
|
mode: 'search',
|
|
site,
|
|
});
|
|
}
|
|
|
|
processes.forEach(entry => {
|
|
entry.logs.forEach(log => {
|
|
if (log.includes('NotFoundError')) {
|
|
const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
|
|
error(ctx, strOut);
|
|
entry.lastError = strOut;
|
|
} else if (log.includes('AuthorizationError')) {
|
|
const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
|
|
error(ctx, strOut);
|
|
entry.lastError = strOut;
|
|
}
|
|
});
|
|
});
|
|
|
|
log(ctx, 'Saving db');
|
|
try {
|
|
let updated = {
|
|
...db,
|
|
userList: {
|
|
...db.userList,
|
|
...Object.fromEntries(processes.map(e => [e.user, userSchema(e)])),
|
|
},
|
|
}
|
|
await writeFile(`${directory}/db.json`, JSON.stringify(updated, null, 2));
|
|
} catch (err) {
|
|
error(ctx, err);
|
|
return;
|
|
}
|
|
log(ctx, 'Done');
|
|
}
|
|
|
|
downloadUsers();
|