split skip params

README.md

@@ -9,7 +9,7 @@ This repo uses its own `config.json` in order to save media in the same format a
 
 ### `node run-downloadDb.js`
 
-Runs a full download of all users listed in the db.json of the archive (the provided `--path`). If db.json is not present, one will be created.
+Runs a full download of all users listed in the db.json of the archive (the provided `--path`). If db.json is not present, one will be created. If any user ends on skipped media during the `/media` check, the `/search` check will be skipped.
 
 Args:
 - `--path={/path/to/your/archive}`
@@ -17,20 +17,25 @@ Args:
 - `--args={gallery-dl args}`
 - `--usersPerBatch={#}`
 - `--waitTime={#}`
+- `--skipMediaAfter={#}`
+- `--skipSearchAfter={#}`
 
 Example:
-- `node run-downloadDb.js --path=/mnt/data/archive --threads=3 --args="-r 2.5M"` will run a full download (`/media` followed by `/search` starting from the oldest pulled file from `/media`) of all the users in the `/mnt/data/archive/db.json` file, limiting concurrent download threads to 3. It will pass the additional args `-r 2.5M --no-skip` to the gallery-dl bin being executed; `-r 2.5M --no-skip` corresponds to limiting the download rate to 2.5M and downloading all files without skipping (for the sake of example).
+- `node run-downloadDb.js --path=/mnt/data/archive --threads=3 --args="-r 2.5M --no-skip"` will run a full download (`/media` followed by `/search` starting from the oldest pulled file from `/media`) of all the users in the `/mnt/data/archive/db.json` file, limiting concurrent download threads to 3. It will pass the additional args `-r 2.5M --no-skip` to the gallery-dl bin being executed; `-r 2.5M --no-skip` corresponds to limiting the download rate to 2.5M and downloading all files without skipping (for the sake of example).
 
 Adding `--usersPerBatch={#}` and `--waitTime={#}` together will activate a batching mechanism which will split the userList in the db.json in chunks of the specified `usersPerBatch` and then wait `waitTime` amount of seconds between each batch in order to throttle any downloads. Without this, 100+ users in a short amount of time could introduce problems, whereas for example ~30 users with ~5 minutes between each batch tends to avoid problems.
 
 ### `run-downloadUsers.js`
 
-Adds new user(s) to the db and initiate a full download similar to `run-downloadDb.js`. If db.json is not present, one will be created.
+Adds new user(s) to the db and initiate a full download similar to `run-downloadDb.js`. If db.json is not present, one will be created. If any user ends on skipped media during the `/media` check, the `/search` check will be skipped.
 
 Args:
-- `--path={/path/to/your/archive}`
 - `--users={comma,separated,userlist}`
+- `--path={/path/to/your/archive}`
 - `--threads={#}`
+- `--args={gallery-dl args}`
+- `--skipMediaAfter={#}`
+- `--skipSearchAfter={#}`
 
 ### `run-convertDb.js`
 
@@ -59,6 +64,14 @@ Max number of concurrent download threads. Only this number of concurrent galler
 
 Additional args to pass to gallery-dl. See [gallery-dl CLI options](https://github.com/mikf/gallery-dl/blob/master/docs/options.md#selection-options) for reference. Note that these aren't currently checked for duplicates that may be used by this repo.
 
+### `--skipMediaAfter={#}`
+
+Appends `-A #` to the args of gallery-dl during the `/media` round, which stops the download early after # amount of skipped media.
+
+### `--skipSearchAfter={#}`
+
+Appends `-A #` to the args of gallery-dl during the `/search` round, which stops the download early after # amount of skipped media.
+
 ## TODO
 
 ### `run-renameUser.js`

lib/dl.js

@@ -118,6 +118,15 @@ export const getUser = (user, path, modeParams) => {
       url += `%20until%3A${modeParams.to}`;
     }
   }
+  let skip;
+  try {
+    skip = modeParams === 'media'
+      ? getArg('skipMediaAfter')
+      : getArg('skipSearchAfter');
+    log(ctx, `Aborting after ${skip} skipped media`);
+  } catch (err) {
+    log(ctx, 'Not aborting after skipped media');
+  }
   let args;
   try {
     args = getArg('args');
@@ -125,8 +134,8 @@ export const getUser = (user, path, modeParams) => {
     log(ctx, 'No args being provided to gallery-dl');
   }
 
-  log(ctx, `python3 ~/.local/bin/gallery-dl -c ./config.json${!!args ? ' ' + args + ' ' : ' '}-d ${path} "${url}"`);
+  log(ctx, `python3 ~/.local/bin/gallery-dl -c ./config.json${!!skip ? ' -A ' + skip : ''}${!!args ? ' ' + args : ''} -d ${path} "${url}"`);
-  const proc = spawn(`python3 ~/.local/bin/gallery-dl -c ./config.json${!!args ? ' ' + args + ' ' : ' '}-d ${path} "${url}"`, { shell: true });
+  const proc = spawn(`python3 ~/.local/bin/gallery-dl -c ./config.json${!!skip ? ' -A ' + skip : ''}${!!args ? ' ' + args : ''} -d ${path} "${url}"`, { shell: true });
 
   proc.stdout.on('data', data => {
     log(ctx, `${data.toString().startsWith('# ') ? '\x1b[90mSkipped ' + data.toString().trim() + '\x1b[0m' : '\x1b[36mDownloaded ' + data.toString().trim() + '\x1b[0m'}`);