Move plugins to express-twtkpr-core-plugins package

Add plugin structure
Fix stale cache after posting
Update to v0.9.0

src/lib/constants.ts

@@ -1,10 +1,11 @@
 import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-export const DEFAULT_PRIVATE_DIRECTORY = '.data';
-export const DEFAULT_PUBLIC_DIRECTORY = 'public';
 export const DEFAULT_TWTXT_FILENAME = 'twtxt.txt';
 export const DEFAULT_ROUTE = `/${DEFAULT_TWTXT_FILENAME}`;
+export const DEFAULT_PRIVATE_DIRECTORY = '.data';
+export const DEFAULT_PUBLIC_DIRECTORY = 'public';
+export const DEFAULT_PLUGIN_ROUTE = '/';
 
 export const DEFAULT_POST_LIMITER_ACTIVE = true;
 

src/lib/env.ts

@@ -3,6 +3,7 @@ import { fileURLToPath } from 'node:url';
 import { z } from 'zod/v4';
 
 import {
+	DEFAULT_PLUGIN_ROUTE,
 	DEFAULT_POST_LIMITER_ACTIVE,
 	DEFAULT_PRIVATE_DIRECTORY,
 	DEFAULT_PUBLIC_DIRECTORY,
@@ -42,6 +43,7 @@ const envSchema = z.object({
 
 	// vars with default values
 	TWTKPR_DEFAULT_ROUTE: z.string().default(DEFAULT_ROUTE),
+	TWTKPR_PLUGIN_ROUTE: z.string().default(DEFAULT_PLUGIN_ROUTE),
 	TWTKPR_PRIVATE_DIRECTORY: z.string().default(DEFAULT_PRIVATE_DIRECTORY),
 	TWTKPR_PUBLIC_DIRECTORY: z.string().default(DEFAULT_PUBLIC_DIRECTORY),
 	TWTKPR_QUERY_PARAMETER_APP: z.string().default(DEFAULT_QUERY_PARAMETER_APP),
@@ -139,6 +141,8 @@ const parseEnv = () => {
 				process.env.TWTKPR_ACCESS_SECRET || process.env.ACCESS_SECRET,
 			TWTKPR_DEFAULT_ROUTE:
 				process.env.TWTKPR_DEFAULT_ROUTE || process.env.DEFAULT_ROUTE,
+			TWTKPR_PLUGIN_ROUTE:
+				process.env.TWTKPR_PLUGIN_ROUTE || process.env.TWTKPR_PLUGIN_ROUTE,
 			TWTKPR_PRIVATE_DIRECTORY:
 				process.env.TWTKPR_PRIVATE_DIRECTORY || process.env.PRIVATE_DIRECTORY,
 			TWTKPR_PUBLIC_DIRECTORY:

src/lib/getConfiguration.ts

@@ -72,12 +72,13 @@ export default function getConfiguration(
 ) {
 	const {
 		mainRoute = env.TWTKPR_DEFAULT_ROUTE,
+		pluginRoute = env.TWTKPR_PLUGIN_ROUTE,
 		privateDirectory = env.TWTKPR_PRIVATE_DIRECTORY,
 		publicDirectory = env.TWTKPR_PUBLIC_DIRECTORY,
 		twtxtFilename = env.TWTKPR_TWTXT_FILENAME,
 		postLimiterConfiguration,
 		queryParameters,
-		uploadConfiguration,
+		// uploadConfiguration,
 	} = initialConfiguration ?? {};
 
 	const {
@@ -96,33 +97,18 @@ export default function getConfiguration(
 		twts = env.TWTKPR_QUERY_PARAMETER_TWTS,
 	} = queryParameters ?? {};
 
-	const {
-		active: uploadActive = env.TWTKPR_UPLOAD_ACTIVE,
-		allowEmptyFiles = env.TWTKPR_UPLOAD_ALLOW_EMPTY_FILES,
-		allowedMimeTypes = env.TWTKPR_UPLOAD_ALLOWED_MIME_TYPES,
-		createDirsFromUploads = env.TWTKPR_UPLOAD_CREATE_DIRS_FROM_UPLOADS,
-		directory = env.TWTKPR_UPLOAD_DIRECTORY,
-		encoding = env.TWTKPR_UPLOAD_ENCODING,
-		fileWriteStreamHandler,
-		filter = () => true,
-		hashAlgorithm = env.TWTKPR_UPLOAD_HASH_ALGORITHM,
-		keepExtensions = env.TWTKPR_UPLOAD_KEEP_EXTENSIONS,
-		maxFields = env.TWTKPR_UPLOAD_MAX_FIELDS,
-		maxFileSize = env.TWTKPR_UPLOAD_MAX_FIELDS_SIZE,
-		maxFiles = env.TWTKPR_UPLOAD_MAX_FILES,
-		maxTotalFileSize = env.TWTKPR_UPLOAD_MAX_TOTAL_FILE_SIZE,
-		minFileSize = env.TWTKPR_UPLOAD_MIN_FILE_SIZE,
-		route = env.TWTKPR_UPLOAD_ROUTE,
-	} = uploadConfiguration ?? {};
-
 	return {
 		// secrets cannot be provided through configuration file, must use ENV / .env
 		accessSecret: env.TWTKPR_ACCESS_SECRET,
 		refreshSecret: env.TWTKPR_REFRESH_SECRET,
 		mainRoute,
+		pluginRoute,
 		privateDirectory,
 		publicDirectory,
 		twtxtFilename,
+		plugins: {
+			...(initialConfiguration?.plugins ?? {}),
+		},
 		postLimiterConfiguration: {
 			active: postLimiterActive,
 			...(otherPostLimiterProps ?? {}),
@@ -138,24 +124,5 @@ export default function getConfiguration(
 			twt,
 			twts,
 		},
-		uploadConfiguration: {
-			...uploadConfiguration,
-			active: uploadActive,
-			allowEmptyFiles,
-			allowedMimeTypes: getDestinationByMimeTypeConfiguration(allowedMimeTypes),
-			createDirsFromUploads,
-			directory,
-			encoding,
-			fileWriteStreamHandler,
-			filter,
-			hashAlgorithm: hashAlgorithm as string | false | undefined,
-			keepExtensions,
-			maxFields,
-			maxFileSize,
-			maxFiles,
-			maxTotalFileSize,
-			minFileSize,
-			route,
-		},
 	} as TwtKprConfiguration;
 }

src/lib/twtxtCache.ts

@@ -1,7 +1,7 @@
 import fsp from 'node:fs/promises';
 import path from 'node:path';
 
-import { NodeCache } from '@cacheable/node-cache';
+import { NodeCache, NodeCacheOptions } from '@cacheable/node-cache';
 import Debug from 'debug';
 
 import { parseTwtxt } from 'twtxt-lib';
@@ -20,9 +20,23 @@ export default function twtxtCache({
 
 	const debug = Debug('twtkpr:twtxtCache');
 
-	const cache = new NodeCache();
+	const defaultCacheOptions: NodeCacheOptions = {
+		stdTTL: 299,
+		checkperiod: 300,
+	};
+
+	const cache = new NodeCache(defaultCacheOptions);
+
+	const getFromCache = async (key = '') => {
+		debug(`checking cache for key: ${key}`);
+		if (!key) return undefined;
+
+		const value = cache.get(key);
+		if (value) return value;
+
+		debug('Not found, reloading keys');
+		cache.flushAll();
 
-	const reloadCache = async () => {
 		const fileText = await fsp.readFile(
 			path.join(publicDirectory, twtxtFilename),
 			'utf8'
@@ -37,12 +51,16 @@ export default function twtxtCache({
 		debug(`cache ${isLoaded ? 're' : ''}loaded`);
 
 		isLoaded = true;
+
+		return cache.get(key);
 	};
 
-	reloadCache();
+	const reloadCache = async () => {
+		cache.flushAll();
+	};
 
 	return {
-		cache,
+		getFromCache,
 		reloadCache,
 	};
 }

src/lib/utils.ts

@@ -1,8 +1,42 @@
 import crypto from 'node:crypto';
+import { createReadStream } from 'node:fs';
 import { readFile, writeFile } from 'node:fs/promises';
+import { PassThrough } from 'node:stream';
+
 import jwt from 'jsonwebtoken';
 import { v4 as uuidv4 } from 'uuid';
 
+import { __dirname } from './constants.js';
+
+async function combineWithPassthrough(sources: any[], destination: any) {
+	for (const stream of sources) {
+		await new Promise((resolve, reject) => {
+			let s = stream;
+			if (typeof stream === 'function') {
+				s = stream();
+			}
+
+			if (typeof s === 'string') {
+				destination.push(s);
+				destination.push(null);
+				resolve(true);
+				return;
+			}
+
+			s.pipe(destination, { end: false });
+			s.on('end', resolve);
+			s.on('error', reject);
+		});
+	}
+	destination.emit('end');
+}
+
+export function combineStreams(streams: any[]) {
+	const stream = new PassThrough();
+	combineWithPassthrough(streams, stream).catch((err) => stream.destroy(err));
+	return stream;
+}
+
 /**
  *
  * @param userId
@@ -51,6 +85,23 @@ export const getQueryParameterArray = (value: unknown | unknown[] = []) =>
 		? value.map((val) => `${val}`.trim())
 		: [`${value}`.trim()];
 
+/**
+ *
+ * @param pathToFile
+ * @returns
+ */
+export const getReadStream = (pathToFile: string) => {
+	const theStream = createReadStream(pathToFile);
+
+	theStream.on('error', (err) => {
+		console.error(err);
+		theStream.close();
+		theStream.push(null);
+	});
+
+	return theStream;
+};
+
 /**
  *
  * @param value