1
0
Fork 0
gallery-dl-archive-manager/run-downloadDb.js

104 lines
2.9 KiB
JavaScript

import { readFile, writeFile } from 'fs/promises';
import { getArg } from './lib/args.js';
import { getMany } from './lib/dl.js';
import { error, log } from './lib/log.js';
import { createDb, userSchema } from './lib/schema.js';
const ctx = 'downloadDb.js';
/**
* Downloads all media possible for the users stored in db.json at the specified `--path`.
* Useful for first run or for augmenting existing media
* if it may be only partially archived in an uncertain state.
*
* If the db.json is missing, it will be automatically created,
* as this depends on users being defined in the db.json and not the folders present
* (as there could be some users whose accounts no longer exist
* or otherwise may not be maintained by the db.json anymore)
*/
const downloadDb = async () => {
log(ctx, 'Grabbing db');
let directory = '', threadMax = 1, db;
try {
directory = getArg('path');
} catch (err) {
error(ctx, err);
return;
}
try {
threadMax = getArg('threads');
log(ctx, `Using ${threadMax} threads`);
} catch (err) {
log(ctx, 'Using 1 thread');
}
const tryReadDb = async () => {
let file = await readFile(`${directory}/db.json`, { encoding: 'utf8' });
db = JSON.parse(file);
}
try {
await tryReadDb();
} catch (err) {
if (err.toString().includes('ENOENT')) {
try {
log(ctx, 'Database was not yet present. Creating it now.');
await createDb();
await tryReadDb();
} catch (err2) {
error(ctx, err2);
return;
}
} else {
error(ctx, err);
return;
}
}
let processes = [];
Object.keys(db.userList).forEach(user => {
processes.push({
...db.userList[user],
user,
lastUpdated: Date.now(),
logs: [],
})
});
log(ctx, `Downloading media using /<user>/media for ${processes.length} users`);
await getMany(processes, threadMax, directory, 'media');
log(ctx, 'Downloading media using /search');
await getMany(processes, threadMax, directory, 'search');
processes.forEach(entry => {
entry.logs.forEach(log => {
if (log.includes('NotFoundError')) {
const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
error(ctx, strOut);
entry.lastError = strOut;
} else if (log.includes('AuthorizationError')) {
const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
error(ctx, strOut);
entry.lastError = strOut;
}
});
});
log(ctx, 'Updating the db');
try {
let updated = {
...db,
userList: {
...db.userList,
...Object.fromEntries(processes.map(e => [e.user, userSchema(e)])),
},
}
await writeFile(`${directory}/db.json`, JSON.stringify(updated, null, 2));
} catch (err) {
error(ctx, err);
return;
}
log(ctx, 'Done');
}
downloadDb();