lightling/gallery-dl-archive-manager
1
0
Fork 0
Code Activity

skip /search if errors or skipped /media

Browse source
This commit is contained in:
lightling 2024-02-28 18:37:09 -05:00
parent 804b1cefc2
commit a20a20b3ce
2 changed files with 26 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
lib/dl.js
View file

@ -52,13 +52,28 @@ export const getMany = (userDb, threadMax, directory, mode) => new Promise((reso
userDb[currentIndex].running = `finished ${mode}`; userDb[currentIndex].running = `finished ${mode}`;
--running; --running;
log(ctx, `Finished (via ${type}) ${userDb[currentIndex].user} under ${mode} mode. ${userDb.filter(elem => elem.running !== `finished ${mode}`).length} users left.`); log(ctx, `Finished (via ${type}) ${userDb[currentIndex].user} under ${mode} mode. ${userDb.filter(elem => elem.running !== `finished ${mode}`).length} users left.`);
if (mode === 'media') {
const logsParsed = userDb[currentIndex].logs.map(buf => buf.toString());
if (logsParsed[logsParsed.length - 2]?.includes('# ')) {
userDb[currentIndex].shouldSkipSearch = true;
}
}
get(); get();
} }
}; };
while (running < threadMax && index < userDb.length) { while (running < threadMax && index < userDb.length) {
++running;
let currentIndex = index++; let currentIndex = index++;
if (mode === 'search') {
if (userDb[currentIndex].shouldSkipSearch) {
log(ctx, `Skipping ${userDb[currentIndex].user} because /media ended with a skipped file`);
continue;
} else if (!!userDb[currentIndex].lastError) {
log(ctx, `Skipping ${userDb[currentIndex].user} because of error: ${userDb[currentIndex].lastError}`);
continue;
}
}
++running;
const modeParams = mode === 'media' ? 'media' : { const modeParams = mode === 'media' ? 'media' : {
from: '2007-12-31', from: '2007-12-31',
to: getDateUrlFromLog(userDb[currentIndex].logs), to: getDateUrlFromLog(userDb[currentIndex].logs),

15
run-downloadDb.js
View file

@ -60,6 +60,7 @@ const downloadDb = async () => {
...db.userList[user], ...db.userList[user],
user, user,
lastUpdated: Date.now(), lastUpdated: Date.now(),
lastError: undefined,
logs: [], logs: [],
}) })
}); });
@ -67,23 +68,24 @@ const downloadDb = async () => {
log(ctx, `Downloading media using /<user>/media for ${processes.length} users`); log(ctx, `Downloading media using /<user>/media for ${processes.length} users`);
await getMany(processes, threadMax, directory, 'media'); await getMany(processes, threadMax, directory, 'media');
log(ctx, 'Downloading media using /search'); const errorReadout = [];
await getMany(processes, threadMax, directory, 'search');
processes.forEach(entry => { processes.forEach(entry => {
entry.logs.forEach(log => { entry.logs.forEach(log => {
if (log.includes('NotFoundError')) { if (log.includes('NotFoundError')) {
const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`; const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
error(ctx, strOut); errorReadout.push(strOut);
entry.lastError = strOut; entry.lastError = strOut;
} else if (log.includes('AuthorizationError')) { } else if (log.includes('AuthorizationError')) {
const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`; const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
error(ctx, strOut); errorReadout.push(strOut);
entry.lastError = strOut; entry.lastError = strOut;
} }
}); });
}); });
log(ctx, 'Downloading media using /search');
await getMany(processes, threadMax, directory, 'search');
log(ctx, 'Updating the db'); log(ctx, 'Updating the db');
try { try {
let updated = { let updated = {
@ -98,6 +100,9 @@ const downloadDb = async () => {
error(ctx, err); error(ctx, err);
return; return;
} }
log(ctx, 'Collecting errors');
errorReadout.forEach(err => error(ctx, err));
log(ctx, 'Done'); log(ctx, 'Done');
} }