skip /search if errors or skipped /media

lib/dl.js

@@ -52,13 +52,28 @@ export const getMany = (userDb, threadMax, directory, mode) => new Promise((reso
         userDb[currentIndex].running = `finished ${mode}`;
         --running;
         log(ctx, `Finished (via ${type}) ${userDb[currentIndex].user} under ${mode} mode. ${userDb.filter(elem => elem.running !== `finished ${mode}`).length} users left.`);
+        if (mode === 'media') {
+          const logsParsed = userDb[currentIndex].logs.map(buf => buf.toString());
+          if (logsParsed[logsParsed.length - 2]?.includes('# ')) {
+            userDb[currentIndex].shouldSkipSearch = true;
+          }
+        }
         get();
       }
     };
 
     while (running < threadMax && index < userDb.length) {
-      ++running;
       let currentIndex = index++;
+      if (mode === 'search') {
+        if (userDb[currentIndex].shouldSkipSearch) {
+          log(ctx, `Skipping ${userDb[currentIndex].user} because /media ended with a skipped file`);
+          continue;
+        } else if (!!userDb[currentIndex].lastError) {
+          log(ctx, `Skipping ${userDb[currentIndex].user} because of error: ${userDb[currentIndex].lastError}`);
+          continue;
+        }
+      }
+      ++running;
       const modeParams = mode === 'media' ? 'media' : {
         from: '2007-12-31',
         to: getDateUrlFromLog(userDb[currentIndex].logs),

run-downloadDb.js

@@ -60,6 +60,7 @@ const downloadDb = async () => {
       ...db.userList[user],
       user,
       lastUpdated: Date.now(),
+      lastError: undefined,
       logs: [],
     })
   });
@@ -67,23 +68,24 @@ const downloadDb = async () => {
   log(ctx, `Downloading media using /<user>/media for ${processes.length} users`);
   await getMany(processes, threadMax, directory, 'media');
 
-  log(ctx, 'Downloading media using /search');
-  await getMany(processes, threadMax, directory, 'search');
-
+  const errorReadout = [];
   processes.forEach(entry => {
     entry.logs.forEach(log => {
       if (log.includes('NotFoundError')) {
         const strOut = `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`;
-        error(ctx, strOut);
+        errorReadout.push(strOut);
         entry.lastError = strOut;
       } else if (log.includes('AuthorizationError')) {
         const strOut = `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`;
-        error(ctx, strOut);
+        errorReadout.push(strOut);
         entry.lastError = strOut;
       }
     });
   });
 
+  log(ctx, 'Downloading media using /search');
+  await getMany(processes, threadMax, directory, 'search');
+
   log(ctx, 'Updating the db');
   try {
     let updated = {
@@ -98,6 +100,9 @@ const downloadDb = async () => {
     error(ctx, err);
     return;
   }
+
+  log(ctx, 'Collecting errors');
+  errorReadout.forEach(err => error(ctx, err));
   log(ctx, 'Done');
 }