cleanup and logging

package-lock.json

@@ -6322,7 +6322,6 @@
       "version": "7.4.1",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
       "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
-      "dev": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -11243,7 +11242,7 @@
       "version": "4.3.5",
       "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.5.tgz",
       "integrity": "sha512-8eabxkth9gZatlwl5TBuJnCsoTADlL6ftEr7A4qgdaTsPyreilDSnUk57SO+jfKcNtxPa22U5KK6DSeAYhpBJw==",
-      "dev": true
+      "devOptional": true
     },
     "node_modules/import-fresh": {
       "version": "3.3.0",
@@ -16516,7 +16515,7 @@
       "version": "1.77.2",
       "resolved": "https://registry.npmjs.org/sass/-/sass-1.77.2.tgz",
       "integrity": "sha512-eb4GZt1C3avsX3heBNlrc7I09nyT00IUuo4eFhAbeXWU2fvA7oXI53SxODVAA+zgZCk9aunAZgO+losjR3fAwA==",
-      "dev": true,
+      "devOptional": true,
       "dependencies": {
         "chokidar": ">=3.0.0 <4.0.0",
         "immutable": "^4.0.0",
@@ -18170,7 +18169,6 @@
       "version": "5.4.5",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
       "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
-      "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -18620,7 +18618,6 @@
       "version": "5.2.11",
       "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.11.tgz",
       "integrity": "sha512-HndV31LWW05i1BLPMUCE1B9E9GFbOu1MbenhS58FuK6owSO5qHm7GiCotrNY1YE5rMeQSFBGmT5ZaLEjFizgiQ==",
-      "dev": true,
       "dependencies": {
         "esbuild": "^0.20.1",
         "postcss": "^8.4.38",
@@ -19689,9 +19686,22 @@
     "projects/sites": {
       "version": "0.0.0",
       "devDependencies": {
+        "chalk": "5.3.0",
         "symlink-dir": "6.0.0"
       }
     },
+    "projects/sites/node_modules/chalk": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz",
+      "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==",
+      "dev": true,
+      "engines": {
+        "node": "^12.17.0 || ^14.13 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
     "projects/types": {
       "version": "0.0.0",
       "extraneous": true

projects/sites/package.json

@@ -4,6 +4,7 @@
   "version": "0.0.0",
   "type": "module",
   "devDependencies": {
+    "chalk": "5.3.0",
     "symlink-dir": "6.0.0"
   },
   "scripts": {

projects/sites/scripts/cms-to-static.js

@@ -1,45 +1,90 @@
-import readlinePromises from 'readline/promises';
+import chalk from 'chalk';
-import { existsSync, createWriteStream } from 'fs';
-import { mkdir, writeFile, unlink } from 'fs/promises';
-import { Readable } from 'stream';
-import { finished } from 'stream/promises';
 
+import readlinePromises from 'readline/promises';
+import { Readable } from 'stream';
+import {
+  existsSync,
+  createWriteStream,
+} from 'fs';
+import { finished } from 'stream/promises';
+import {
+  mkdir,
+  unlink,
+  writeFile,
+} from 'fs/promises';
+
+import { log } from './log.js';
+
+/**
+ * Validates a string as a MediaEntry id
+ * @param {string} str gets a valid id for a MediaEntry; it cannot have symbols that are not valid in paths
+ * @returns {string} the validated string
+ */
 const getValidId = (str) => (str || '').replace(/( |\/|\\)/gm,'_');
 
+/**
+ * Saves the given url to the given destination
+ * @param {string} url the url of the file to save
+ * @param {string} destination the file destination
+ */
+const saveFileFromUrl = async (url, destination) => {
+  if (existsSync(destination)) {
+    log(`Deleting existing file ${chalk.italic(destination)}`, 'debug');
+    await unlink(destination);
+    log(`Deleted file ${chalk.italic(destination)}`, 'info');
+  }
+
+  log(`Downloading file ${url}`, 'debug');
+  const res = await fetch(url);
+  log(`Saving file ${chalk.italic(url)} to ${chalk.italic(destination)}`, 'debug');
+  const fileStream = createWriteStream(destination, { flags: 'wx' });
+  await finished(Readable.fromWeb(res.body).pipe(fileStream));
+  log(`File ${chalk.italic(url)} saved to ${chalk.italic(destination)}`, 'info');
+};
+
+/**
+ * Saves a media list's entries to the given destination and returns an updated list where the Strapi urls have been replaced
+ * @param {MediaList} inVal the media list to save files from
+ * @param {string} fetchUrl the Strapi API fetch url
+ * @param {string} destination the parent destination where the media list is saving files/folders in
+ * @param {string} destinationShort the shortened destination, where 'sites/{my-site}/' has been removed so that the file can be '/content/{file/path}'
+ * @returns {Promise<MediaList>} the updated media list
+ */
 const saveVariantsToDestination = async (inVal, fetchUrl, destination, destinationShort) => {
   const rootUrl = fetchUrl.replace(/((http)|(https)):\/\//, '').split('/')[0];
   const outVal = JSON.parse(JSON.stringify(inVal));
+  // Loop over each entry in the media list
   await Promise.all(Object.keys(outVal.entries).map(async entryId => {
-    const entryPath = `${destination}/${entryId}`
+    const entryPath = `${destination}/${entryId}`;
     if (!existsSync(entryPath)) {
+      log(`Creating directory at ${chalk.italic(entryPath)}`, 'debug');
       await mkdir(entryPath);
+      log(`Created directory ${chalk.italic(entryPath)}`, 'info');
     }
+
+    // Loop over each variant in the media entry
     await Promise.all(outVal.entries[entryId].variants.map(async (variant, index) => {
-      const variantPath = `${entryPath}/${variant.url.replace('/uploads/', '')}`;
+      // save the main file and update in the variant info
-      if (existsSync(variantPath)) {
+      const filePath = `${entryPath}/${variant.url.replace('/uploads/', '')}`;
-        await unlink(variantPath);
+      saveFileFromUrl(`http://${rootUrl}${variant.url}`, filePath);
-      }
-      const variantFile = await fetch(`http://${rootUrl}${variant.url}`);
-      const variantStream = createWriteStream(variantPath, { flags: 'wx' });
-      await finished(Readable.fromWeb(variantFile.body).pipe(variantStream));
-
-      const thumbnailPath = `${entryPath}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
-      if (existsSync(thumbnailPath)) {
-        await unlink(thumbnailPath);
-      }
-      const thumbnailFile = await fetch(`http://${rootUrl}${variant.thumbnailUrl}`);
-      const thumbnailStream = createWriteStream(thumbnailPath, { flags: 'wx' });
-      await finished(Readable.fromWeb(thumbnailFile.body).pipe(thumbnailStream));
-
       outVal.entries[entryId].variants[index].url = `/content/${destinationShort}/${entryId}/${variant.url.replace('/uploads/', '')}`;
+
+      // save the thumbnail and update in the variant info
+      const thumbnailPath = `${entryPath}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
+      saveFileFromUrl(`http://${rootUrl}${variant.thumbnailUrl}`, thumbnailPath);
       outVal.entries[entryId].variants[index].thumbnailUrl = `/content/${destinationShort}/${entryId}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
     }));
   }));
 
-  return outVal;
+  return new Promise(resolve => resolve(outVal));
 };
 
-const mapper = (inVal) => {
+/**
+ * Maps Strapi API response to a mackenzii media list
+ * @param {StrapiMediaList} inVal the Strapi API response
+ * @returns {Promise<MediaList>} the media list mapped from the Strapi API
+ */
+const mapStrapiResponseToMackenzii = async (inVal) => {
   const { data } = inVal;
   const outVal = {};
 
@@ -76,7 +121,7 @@ const mapper = (inVal) => {
     outVal.entries[id].tags = (outVal.entries[id].tags || '').split(/,| |;/).filter(val => val !== '');
   });
 
-  return outVal;
+  return new Promise(resolve => resolve(outVal));
 };
 
 (async () => {
@@ -85,10 +130,10 @@ const mapper = (inVal) => {
   try {
     fetchUrl = await rl.question('Enter the URL to fetch from: ');
   } catch (err) {
-    console.error('There was an error: ', err);
+    log('There was an error: ', 'error', 'cms-to-static::main::Q1', err);
   } finally {
     if (!fetchUrl) {
-      console.log('Invalid URL. Quitting...');
+      log('An invalid URL was given. Quitting...', 'log', 'cms-to-static::main::Q1');
       rl.close();
       return;
     } else if (!fetchUrl.includes('?')) {
@@ -101,11 +146,11 @@ const mapper = (inVal) => {
     destination = await rl.question('Enter the directory to save the response\n(note: this should be a path relative to "sites";\n{your/input} will become sites/{your/input}):');
     destination = 'sites/' + destination;
   } catch (err) {
-    console.error('There was an error: ', err);
+    log('There was an error: ', 'error', 'cms-to-static::main::Q2', err);
   } finally {
     rl.close();
     if (!destination) {
-      console.log('Invalid path. Quitting...');
+      log('An invalid path was given. Quitting...', 'log', 'cms-to-static::main::Q2');
       return;
     } else {
       if (!existsSync(destination)) {
@@ -123,16 +168,18 @@ const mapper = (inVal) => {
       },
     })).json();
   } catch (err) {
-    console.error('There was an error: ', err);
+    log('There was an error: ', 'error', 'cms-to-static::main', err);
   }
 
-  console.log('Writing...');
+  log('Writing...', 'log', 'cms-to-static::main::writing');
   try {
     await writeFile(`${destination}/in.json`, JSON.stringify(res, null, 2));
-    let list = mapper(res);
+
-    list = await saveVariantsToDestination(list, fetchUrl, destination, destination.replace('sites/', '').split('/').slice(1).join('/'));
+    mapStrapiResponseToMackenzii(res)
-    await writeFile(`${destination}/out.json`, JSON.stringify(list, null, 2));
+      .then(list => saveVariantsToDestination(list, fetchUrl, destination, destination.replace('sites/', '').split('/').slice(1).join('/')))
+      .then(list => writeFile(`${destination}/out.json`, JSON.stringify(list, null, 2)));
+    log('Done!', 'log', 'cms-to-static::main::writing');
   } catch (err) {
-    console.error('There was an error: ', err)
+    log('There was an error: ', 'error', 'cms-to-static::main::writing', err);
   }
 })();

projects/sites/scripts/log.js

@@ -0,0 +1,103 @@
+import chalk from 'chalk';
+
+/**
+ * Formats the given date for logging
+ * @param {Date} date the date to format
+ * @returns {string} the formatted date string in yyyy/mm/dd hh/mm/ss/SSS
+ */
+export const time = (date) => {
+  /**
+   * Stringifies the given number, prepends with 0s if the number string isn't long enough
+   * @param {number} num the number to stringify
+   * @param {number} expectedLength how long the number should be
+   * @returns {string} the stringified number
+   */
+  const stringifyNumber = (num, expectedLength) => {
+    let str = `${num}`;
+    while (str.length < expectedLength) {
+      str = `0${str}`;
+    }
+    return str;
+  };
+
+  const year = date.getFullYear()
+  const month = stringifyNumber(date.getMonth(), 2);
+  const day = stringifyNumber(date.getDay(), 2);
+  const hour = stringifyNumber(date.getHours(), 2);
+  const minute = stringifyNumber(date.getMinutes(), 2);
+  const second = stringifyNumber(date.getSeconds(), 2);
+  const millisecond = stringifyNumber(date.getMilliseconds(), 2);
+
+  return `${year}/${month}/${day} ${hour}:${minute}:${second}.${millisecond}`;
+}
+
+/**
+ * Logs a message to the console
+ * @param {string} message the message to log
+ * @param {'debug'|'info'|'log'|'warn'|'error'} kind the kind of message being logged; denotes logging level
+ * - debug is lowest level of message (0)
+ * - info is the second lowest level of message (1)
+ * - log is the default level level of message (2)
+ * - warn is the second highest level of message (3)
+ * - error is the highest level of message (4)
+ * @param {string} context the context; if undefined, assume part of a parent context and indent the message
+ * @param {string} extraData any extra data (e.g. if there was an error, pass the error object here)
+ */
+export const log = (message, kind, context, extraData) => {
+  const logLevel = process.env.npm_config_logging || 2;
+  const formatted = `${chalk.bold(`[${time(new Date())}] `)}${context ? context + ': ' : '  '}${message}`;
+  switch (kind) {
+    case 'debug': {
+      if (logLevel > 0) {
+        return;
+      }
+      if (extraData) {
+        console.debug(`${chalk.gray.bold('[debug]')}${formatted}`, extraData);
+      } else {
+        console.debug(`${chalk.gray.bold('[debug]')}${formatted}`);
+      }
+      break;
+    }
+    case 'info': {
+      if (logLevel > 1) {
+        return;
+      }
+      if (extraData) {
+        console.info(`${chalk.white.bold('[info]')}${formatted}`, extraData);
+      } else {
+        console.info(`${chalk.white.bold('[info]')}${formatted}`);
+      }
+      break;
+    }
+    case 'warn': {
+      if (logLevel > 3) {
+        return;
+      }
+      if (extraData) {
+        console.warn(`${chalk.yellow.bold('[warn]')}${formatted}`, extraData);
+      } else {
+        console.warn(`${chalk.yellow.bold('[warn]')}${formatted}`);
+      }
+      break;
+    }
+    case 'error': {
+      if (extraData) {
+        console.error(`${chalk.red.bold('[error]')}${formatted}`, extraData);
+      } else {
+        console.error(`${chalk.red.bold('[error]')}${formatted}`);
+      }
+      break;
+    }
+    default: {
+      if (logLevel > 2) {
+        return;
+      }
+      if (extraData) {
+        console.log(`${chalk.blue.bold('[log]')}${formatted}`, extraData);
+      } else {
+        console.log(`${chalk.blue.bold('[log]')}${formatted}`);
+      }
+      break;
+    }
+  }
+};