save images locally

projects/sites/scripts/cms-to-static.js

@@ -1,6 +1,43 @@
 import readlinePromises from 'readline/promises';
-import { existsSync } from 'fs';
+import { existsSync, createWriteStream } from 'fs';
-import { mkdir, writeFile } from 'fs/promises';
+import { mkdir, writeFile, unlink } from 'fs/promises';
+import { Readable } from 'stream';
+import { finished } from 'stream/promises';
+
+const getValidId = (str) => (str || '').replace(/( |\/|\\)/gm,'_');
+
+const saveVariantsToDestination = async (inVal, fetchUrl, destination, destinationShort) => {
+  const rootUrl = fetchUrl.replace(/((http)|(https)):\/\//, '').split('/')[0];
+  const outVal = JSON.parse(JSON.stringify(inVal));
+  await Promise.all(Object.keys(outVal.entries).map(async entryId => {
+    const entryPath = `${destination}/${entryId}`
+    if (!existsSync(entryPath)) {
+      await mkdir(entryPath);
+    }
+    await Promise.all(outVal.entries[entryId].variants.map(async (variant, index) => {
+      const variantPath = `${entryPath}/${variant.url.replace('/uploads/', '')}`;
+      if (existsSync(variantPath)) {
+        await unlink(variantPath);
+      }
+      const variantFile = await fetch(`http://${rootUrl}${variant.url}`);
+      const variantStream = createWriteStream(variantPath, { flags: 'wx' });
+      await finished(Readable.fromWeb(variantFile.body).pipe(variantStream));
+
+      const thumbnailPath = `${entryPath}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
+      if (existsSync(thumbnailPath)) {
+        await unlink(thumbnailPath);
+      }
+      const thumbnailFile = await fetch(`http://${rootUrl}${variant.thumbnailUrl}`);
+      const thumbnailStream = createWriteStream(thumbnailPath, { flags: 'wx' });
+      await finished(Readable.fromWeb(thumbnailFile.body).pipe(thumbnailStream));
+
+      outVal.entries[entryId].variants[index].url = `/content/${destinationShort}/${entryId}/${variant.url.replace('/uploads/', '')}`;
+      outVal.entries[entryId].variants[index].thumbnailUrl = `/content/${destinationShort}/${entryId}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
+    }));
+  }));
+
+  return outVal;
+};
 
 const mapper = (inVal) => {
   const { data } = inVal;
@@ -13,7 +50,7 @@ const mapper = (inVal) => {
   outVal.entries = {};
   data.entries.forEach(entry => {
     // give it an id
-    const id = `${(entry.date || '').replace(/ /gm,'_')}_${(entry.title || '').replace(/ /gm, '_')}_${entry.documentId}`;
+    const id = `${getValidId(entry.date)}_${getValidId(entry.title)}_${getValidId(entry.documentId)}`;
 
     // copy all values at first...
     outVal.entries[id] = {...entry};
@@ -61,7 +98,8 @@ const mapper = (inVal) => {
 
   let destination = '';
   try {
-    destination = await rl.question('Enter the directory to save the response (note: start from sites/{your-site-here} for relative pathing): ');
+    destination = await rl.question('Enter the directory to save the response\n(note: this should be a path relative to "sites";\n{your/input} will become sites/{your/input}):');
+    destination = 'sites/' + destination;
   } catch (err) {
     console.error('There was an error: ', err);
   } finally {
@@ -71,7 +109,7 @@ const mapper = (inVal) => {
       return;
     } else {
       if (!existsSync(destination)) {
-        await mkdir(destination);
+        await mkdir(destination, { recursive: true });
       }
     }
   }
@@ -92,6 +130,7 @@ const mapper = (inVal) => {
   try {
     await writeFile(`${destination}/in.json`, JSON.stringify(res, null, 2));
     let list = mapper(res);
+    list = await saveVariantsToDestination(list, fetchUrl, destination, destination.replace('sites/', '').split('/').slice(1).join('/'));
     await writeFile(`${destination}/out.json`, JSON.stringify(list, null, 2));
   } catch (err) {
     console.error('There was an error: ', err)