Goldenwere/mackenzii
Goldenwere/mackenzii
1
0
Fork 0
Code Activity Actions

cleanup and logging

Browse source
This commit is contained in:
lightling 2024-10-16 01:58:05 -04:00
parent c89ade7018
commit 916b029aa5
4 changed files with 201 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
package-lock.json generated
View file

@ -6322,7 +6322,6 @@
"version": "7.4.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
"integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
"dev": true,
"bin": {
"acorn": "bin/acorn"
},
@ -11243,7 +11242,7 @@
"version": "4.3.5",
"resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.5.tgz",
"integrity": "sha512-8eabxkth9gZatlwl5TBuJnCsoTADlL6ftEr7A4qgdaTsPyreilDSnUk57SO+jfKcNtxPa22U5KK6DSeAYhpBJw==",
"dev": true
"devOptional": true
},
"node_modules/import-fresh": {
"version": "3.3.0",
@ -16516,7 +16515,7 @@
"version": "1.77.2",
"resolved": "https://registry.npmjs.org/sass/-/sass-1.77.2.tgz",
"integrity": "sha512-eb4GZt1C3avsX3heBNlrc7I09nyT00IUuo4eFhAbeXWU2fvA7oXI53SxODVAA+zgZCk9aunAZgO+losjR3fAwA==",
"dev": true,
"devOptional": true,
"dependencies": {
"chokidar": ">=3.0.0 <4.0.0",
"immutable": "^4.0.0",
@ -18170,7 +18169,6 @@
"version": "5.4.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
"integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@ -18620,7 +18618,6 @@
"version": "5.2.11",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.2.11.tgz",
"integrity": "sha512-HndV31LWW05i1BLPMUCE1B9E9GFbOu1MbenhS58FuK6owSO5qHm7GiCotrNY1YE5rMeQSFBGmT5ZaLEjFizgiQ==",
"dev": true,
"dependencies": {
"esbuild": "^0.20.1",
"postcss": "^8.4.38",
@ -19689,9 +19686,22 @@
"projects/sites": {
"version": "0.0.0",
"devDependencies": {
"chalk": "5.3.0",
"symlink-dir": "6.0.0"
}
},
"projects/sites/node_modules/chalk": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz",
"integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==",
"dev": true,
"engines": {
"node": "^12.17.0 || ^14.13 || >=16.0.0"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"projects/types": {
"version": "0.0.0",
"extraneous": true

1
projects/sites/package.json
View file

@ -4,6 +4,7 @@
"version": "0.0.0",
"type": "module",
"devDependencies": {
"chalk": "5.3.0",
"symlink-dir": "6.0.0"
},
"scripts": {

117
projects/sites/scripts/cms-to-static.js
View file

@ -1,45 +1,90 @@
import readlinePromises from 'readline/promises';
import { existsSync, createWriteStream } from 'fs';
import { mkdir, writeFile, unlink } from 'fs/promises';
import { Readable } from 'stream';
import { finished } from 'stream/promises';
import chalk from 'chalk';
import readlinePromises from 'readline/promises';
import { Readable } from 'stream';
import {
existsSync,
createWriteStream,
} from 'fs';
import { finished } from 'stream/promises';
import {
mkdir,
unlink,
writeFile,
} from 'fs/promises';
import { log } from './log.js';
/**
* Validates a string as a MediaEntry id
* @param {string} str gets a valid id for a MediaEntry; it cannot have symbols that are not valid in paths
* @returns {string} the validated string
*/
const getValidId = (str) => (str || '').replace(/( |\/|\\)/gm,'_');
/**
* Saves the given url to the given destination
* @param {string} url the url of the file to save
* @param {string} destination the file destination
*/
const saveFileFromUrl = async (url, destination) => {
if (existsSync(destination)) {
log(`Deleting existing file ${chalk.italic(destination)}`, 'debug');
await unlink(destination);
log(`Deleted file ${chalk.italic(destination)}`, 'info');
}
log(`Downloading file ${url}`, 'debug');
const res = await fetch(url);
log(`Saving file ${chalk.italic(url)} to ${chalk.italic(destination)}`, 'debug');
const fileStream = createWriteStream(destination, { flags: 'wx' });
await finished(Readable.fromWeb(res.body).pipe(fileStream));
log(`File ${chalk.italic(url)} saved to ${chalk.italic(destination)}`, 'info');
};
/**
* Saves a media list's entries to the given destination and returns an updated list where the Strapi urls have been replaced
* @param {MediaList} inVal the media list to save files from
* @param {string} fetchUrl the Strapi API fetch url
* @param {string} destination the parent destination where the media list is saving files/folders in
* @param {string} destinationShort the shortened destination, where 'sites/{my-site}/' has been removed so that the file can be '/content/{file/path}'
* @returns {Promise<MediaList>} the updated media list
*/
const saveVariantsToDestination = async (inVal, fetchUrl, destination, destinationShort) => {
const rootUrl = fetchUrl.replace(/((http)|(https)):\/\//, '').split('/')[0];
const outVal = JSON.parse(JSON.stringify(inVal));
// Loop over each entry in the media list
await Promise.all(Object.keys(outVal.entries).map(async entryId => {
const entryPath = `${destination}/${entryId}`
const entryPath = `${destination}/${entryId}`;
if (!existsSync(entryPath)) {
log(`Creating directory at ${chalk.italic(entryPath)}`, 'debug');
await mkdir(entryPath);
log(`Created directory ${chalk.italic(entryPath)}`, 'info');
}
// Loop over each variant in the media entry
await Promise.all(outVal.entries[entryId].variants.map(async (variant, index) => {
const variantPath = `${entryPath}/${variant.url.replace('/uploads/', '')}`;
if (existsSync(variantPath)) {
await unlink(variantPath);
}
const variantFile = await fetch(`http://${rootUrl}${variant.url}`);
const variantStream = createWriteStream(variantPath, { flags: 'wx' });
await finished(Readable.fromWeb(variantFile.body).pipe(variantStream));
const thumbnailPath = `${entryPath}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
if (existsSync(thumbnailPath)) {
await unlink(thumbnailPath);
}
const thumbnailFile = await fetch(`http://${rootUrl}${variant.thumbnailUrl}`);
const thumbnailStream = createWriteStream(thumbnailPath, { flags: 'wx' });
await finished(Readable.fromWeb(thumbnailFile.body).pipe(thumbnailStream));
// save the main file and update in the variant info
const filePath = `${entryPath}/${variant.url.replace('/uploads/', '')}`;
saveFileFromUrl(`http://${rootUrl}${variant.url}`, filePath);
outVal.entries[entryId].variants[index].url = `/content/${destinationShort}/${entryId}/${variant.url.replace('/uploads/', '')}`;
// save the thumbnail and update in the variant info
const thumbnailPath = `${entryPath}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
saveFileFromUrl(`http://${rootUrl}${variant.thumbnailUrl}`, thumbnailPath);
outVal.entries[entryId].variants[index].thumbnailUrl = `/content/${destinationShort}/${entryId}/${variant.thumbnailUrl.replace('/uploads/', '')}`;
}));
}));
return outVal;
return new Promise(resolve => resolve(outVal));
};
const mapper = (inVal) => {
/**
* Maps Strapi API response to a mackenzii media list
* @param {StrapiMediaList} inVal the Strapi API response
* @returns {Promise<MediaList>} the media list mapped from the Strapi API
*/
const mapStrapiResponseToMackenzii = async (inVal) => {
const { data } = inVal;
const outVal = {};
@ -76,7 +121,7 @@ const mapper = (inVal) => {
outVal.entries[id].tags = (outVal.entries[id].tags || '').split(/,| |;/).filter(val => val !== '');
});
return outVal;
return new Promise(resolve => resolve(outVal));
};
(async () => {
@ -85,10 +130,10 @@ const mapper = (inVal) => {
try {
fetchUrl = await rl.question('Enter the URL to fetch from: ');
} catch (err) {
console.error('There was an error: ', err);
log('There was an error: ', 'error', 'cms-to-static::main::Q1', err);
} finally {
if (!fetchUrl) {
console.log('Invalid URL. Quitting...');
log('An invalid URL was given. Quitting...', 'log', 'cms-to-static::main::Q1');
rl.close();
return;
} else if (!fetchUrl.includes('?')) {
@ -101,11 +146,11 @@ const mapper = (inVal) => {
destination = await rl.question('Enter the directory to save the response\n(note: this should be a path relative to "sites";\n{your/input} will become sites/{your/input}):');
destination = 'sites/' + destination;
} catch (err) {
console.error('There was an error: ', err);
log('There was an error: ', 'error', 'cms-to-static::main::Q2', err);
} finally {
rl.close();
if (!destination) {
console.log('Invalid path. Quitting...');
log('An invalid path was given. Quitting...', 'log', 'cms-to-static::main::Q2');
return;
} else {
if (!existsSync(destination)) {
@ -123,16 +168,18 @@ const mapper = (inVal) => {
},
})).json();
} catch (err) {
console.error('There was an error: ', err);
log('There was an error: ', 'error', 'cms-to-static::main', err);
}
console.log('Writing...');
log('Writing...', 'log', 'cms-to-static::main::writing');
try {
await writeFile(`${destination}/in.json`, JSON.stringify(res, null, 2));
let list = mapper(res);
list = await saveVariantsToDestination(list, fetchUrl, destination, destination.replace('sites/', '').split('/').slice(1).join('/'));
await writeFile(`${destination}/out.json`, JSON.stringify(list, null, 2));
mapStrapiResponseToMackenzii(res)
.then(list => saveVariantsToDestination(list, fetchUrl, destination, destination.replace('sites/', '').split('/').slice(1).join('/')))
.then(list => writeFile(`${destination}/out.json`, JSON.stringify(list, null, 2)));
log('Done!', 'log', 'cms-to-static::main::writing');
} catch (err) {
console.error('There was an error: ', err)
log('There was an error: ', 'error', 'cms-to-static::main::writing', err);
}
})();

103
projects/sites/scripts/log.js Normal file
View file

@ -0,0 +1,103 @@
import chalk from 'chalk';
/**
* Formats the given date for logging
* @param {Date} date the date to format
* @returns {string} the formatted date string in yyyy/mm/dd hh/mm/ss/SSS
*/
export const time = (date) => {
/**
* Stringifies the given number, prepends with 0s if the number string isn't long enough
* @param {number} num the number to stringify
* @param {number} expectedLength how long the number should be
* @returns {string} the stringified number
*/
const stringifyNumber = (num, expectedLength) => {
let str = `${num}`;
while (str.length < expectedLength) {
str = `0${str}`;
}
return str;
};
const year = date.getFullYear()
const month = stringifyNumber(date.getMonth(), 2);
const day = stringifyNumber(date.getDay(), 2);
const hour = stringifyNumber(date.getHours(), 2);
const minute = stringifyNumber(date.getMinutes(), 2);
const second = stringifyNumber(date.getSeconds(), 2);
const millisecond = stringifyNumber(date.getMilliseconds(), 2);
return `${year}/${month}/${day} ${hour}:${minute}:${second}.${millisecond}`;
}
/**
* Logs a message to the console
* @param {string} message the message to log
* @param {'debug'|'info'|'log'|'warn'|'error'} kind the kind of message being logged; denotes logging level
* - debug is lowest level of message (0)
* - info is the second lowest level of message (1)
* - log is the default level level of message (2)
* - warn is the second highest level of message (3)
* - error is the highest level of message (4)
* @param {string} context the context; if undefined, assume part of a parent context and indent the message
* @param {string} extraData any extra data (e.g. if there was an error, pass the error object here)
*/
export const log = (message, kind, context, extraData) => {
const logLevel = process.env.npm_config_logging || 2;
const formatted = `${chalk.bold(`[${time(new Date())}] `)}${context ? context + ': ' : ' '}${message}`;
switch (kind) {
case 'debug': {
if (logLevel > 0) {
return;
}
if (extraData) {
console.debug(`${chalk.gray.bold('[debug]')}${formatted}`, extraData);
} else {
console.debug(`${chalk.gray.bold('[debug]')}${formatted}`);
}
break;
}
case 'info': {
if (logLevel > 1) {
return;
}
if (extraData) {
console.info(`${chalk.white.bold('[info]')}${formatted}`, extraData);
} else {
console.info(`${chalk.white.bold('[info]')}${formatted}`);
}
break;
}
case 'warn': {
if (logLevel > 3) {
return;
}
if (extraData) {
console.warn(`${chalk.yellow.bold('[warn]')}${formatted}`, extraData);
} else {
console.warn(`${chalk.yellow.bold('[warn]')}${formatted}`);
}
break;
}
case 'error': {
if (extraData) {
console.error(`${chalk.red.bold('[error]')}${formatted}`, extraData);
} else {
console.error(`${chalk.red.bold('[error]')}${formatted}`);
}
break;
}
default: {
if (logLevel > 2) {
return;
}
if (extraData) {
console.log(`${chalk.blue.bold('[log]')}${formatted}`, extraData);
} else {
console.log(`${chalk.blue.bold('[log]')}${formatted}`);
}
break;
}
}
};