better error handling

- rate limit is stderr, so need to check to make sure only finishing on other errors
- color errors in log
- log errors to userDb
- use fallback date in case dates weren't logged (e.g. /media failed)
- report on not-found users, which may indicate a username change or deleted account
- report on authorization errors

lib/dl.js

@@ -20,6 +20,11 @@ const getDateUrlFromLog = (logs) => {
     result = last;
     last = loggedDateRegex.exec(flat);
   }
+  if (!result) {
+    let fallbackDate = new Date();
+
+    return `${fallbackDate.getUTCFullYear()}-${fallbackDate.getUTCMonth()}-${fallbackDate.getUTCDate()}`
+  }
   let date = new Date(result[1], result[2], result[3], result[4], result[5], result[6], 0);
   // to be safe, spring forward a day
   date.setDate(date.getDate() + 1);
@@ -40,11 +45,19 @@ export const getMany = (userDb, threadMax, directory, mode) => new Promise((reso
   let index = 0;
 
   const get = () => {
-    const onFinish = (currentIndex) => {
-      log(ctx, `Finished ${userDb[currentIndex].user} under ${mode} mode`);
+    const checkError = (proc, currentIndex, type, codeOrError) => {
+      userDb[currentIndex].logs.push(codeOrError.toString())
+      if (!!proc.exitCode && !userDb[currentIndex].running) {
+        userDb[currentIndex].running = false;
+        onFinish(currentIndex, type);
+      }
+    };
+
+    const onFinish = (currentIndex, type) => {
+      log(ctx, `Finished ${userDb[currentIndex].user} under ${mode} mode.${type === 'close' ? '' : ' (Closed due to ' + type}`);
       --running;
       get();
-    }
+    };
 
     while (running < threadMax && index < userDb.length) {
       ++running;
@@ -55,12 +68,14 @@ export const getMany = (userDb, threadMax, directory, mode) => new Promise((reso
       };
 
       let proc = getUser(userDb[currentIndex].user, directory, modeParams);
+      userDb[currentIndex].running = true;
       proc.stdout.on('data', data => {
         userDb[currentIndex].logs.push(data);
       });
-      proc.stderr.on('data', _ => onFinish(currentIndex));
-      proc.on('close', _ => onFinish(currentIndex));
-      proc.on('error', _ => onFinish(currentIndex));
+      proc.stderr.on('data', data => checkError(proc, currentIndex, 'stderr', data));
+      proc.on('close', code => onFinish(currentIndex, 'close', code));
+      proc.on('error', err => checkError(proc, currentIndex, 'error', err));
+      proc.on('exit', code => checkError(proc, currentIndex, 'exit', code));
     }
 
     if (running === 0) {

lib/log.js

@@ -27,5 +27,5 @@ export const log = (src, msg) => {
  */
 export const error = (src, msg) => {
   const time = new Date().toISOString();
-  console.error(`${time} : ${src} : ${msg}`);
+  console.error(`\x1b[33m${time} : ${src} : ${msg}\x1b[0m`);
 }

run-downloadDb.js

@@ -45,6 +45,15 @@ const downloadDb = async () => {
   log(ctx, 'Downloading media using /search');
   await getMany(processes, threadMax, directory, 'search');
 
+  processes.forEach(entry => {
+    entry.logs.forEach(log => {
+      if (log.includes('NotFoundError')) {
+        error(ctx, `${entry.user} wasn't found: "${log.replace('\n', '')}". You may want to remove them from the db.json file or update their username.`);
+      } else if (log.includes('AuthorizationError')) {
+        error(ctx, `There was an authorization error for user ${entry.user}: "${log.replace('\n', '')}"`);
+      }
+    });
+  });
   log(ctx, 'Done');
 }