Move plugins to express-twtkpr-core-plugins package

Add plugin structure
Fix stale cache after posting
Update to v0.9.0

src/middlewares/queryHandler/followingHandler.ts

@@ -19,7 +19,7 @@ import NodeCache from '@cacheable/node-cache';
 export default function followingHandler(
 	req: Request,
 	res: Response,
-	cache: NodeCache<unknown>,
+	following: Twttr[],
 	followingParameter: QueryParameters['following']
 ) {
 	const followingsToMatch = getQueryParameterArray(
@@ -40,7 +40,7 @@ export default function followingHandler(
 	if (wantsJson) res.set('content-type', 'application/json');
 	else res.set('content-type', 'text/plain');
 
-	const matchedFollowing = (cache.get('following') as Twttr[]).filter(
+	const matchedFollowing = following.filter(
 		({ nick, url }) =>
 			(!followingsToMatch.length ||
 				(followingsToMatch.length === 1 && followingsToMatch[0] === '') ||

src/middlewares/queryHandler/metadataHandler.ts

@@ -1,14 +1,12 @@
 import type { Request, Response } from 'express';
 import type { Metadata } from 'twtxt-lib';
 
-import { env } from '../../lib/env.js';
-import twtxtCache from '../../lib/twtxtCache.js';
 import {
 	generateEtag,
 	getQueryParameterArray,
 	getValueOrFirstEntry,
 } from '../../lib/utils.js';
-import NodeCache from '@cacheable/node-cache';
+import { NodeCache } from '@cacheable/node-cache';
 import { QueryParameters } from '../../types.js';
 
 export interface MetadataHandler {
@@ -28,7 +26,7 @@ export interface MetadataHandler {
 export default function metadataHandler(
 	req: Request,
 	res: Response,
-	cache: NodeCache<unknown>,
+	metadata: Metadata,
 	metadataParameter: QueryParameters['metadata']
 ) {
 	const metadataToMatch = getQueryParameterArray(req.query[metadataParameter]);
@@ -38,8 +36,6 @@ export default function metadataHandler(
 		...getQueryParameterArray(req.query.s),
 	];
 
-	const metadata = (cache.get('metadata') as Metadata) ?? {};
-
 	const wantsJson =
 		req.is('json') ||
 		getValueOrFirstEntry(getQueryParameterArray(req.query.format)) === 'json';

src/middlewares/queryHandler/queryHandler.ts

@@ -1,15 +1,15 @@
+import fs from 'node:fs';
 import path from 'node:path';
-import NodeCache from '@cacheable/node-cache';
 import Debug from 'debug';
 import type { NextFunction, Request, Response } from 'express';
 import { __dirname } from '../../lib/constants.js';
-import { generateEtag } from '../../lib/utils.js';
-import { TwtKprConfiguration } from '../../types.js';
+import { combineStreams, generateEtag } from '../../lib/utils.js';
+import { TwtKprConfiguration, TwtKprPluginConfiguration } from '../../types.js';
 import renderApp from '../renderApp/index.js';
 import followingHandler from './followingHandler.js';
 import metadataHandler from './metadataHandler.js';
 import twtHandler from './twtHandler.js';
-import { Twt } from 'twtxt-lib';
+import { Metadata, Twt, Twttr } from 'twtxt-lib';
 
 const debug = Debug('twtkpr:queryHandler');
 
@@ -22,13 +22,17 @@ const debug = Debug('twtkpr:queryHandler');
  */
 export default function queryHandler(
 	config: TwtKprConfiguration,
-	cache: NodeCache<unknown>,
-	verifyAuthRequest: (r: Request) => Promise<boolean>
+	getFromCache: (key?: string) => Promise<unknown>,
+	verifyAuthRequest: (r: Request) => Promise<boolean>,
+	plugins: TwtKprPluginConfiguration[] = []
 ) {
-	const { mainRoute, queryParameters, uploadConfiguration } = config;
+	const { mainRoute, queryParameters } = config;
+
+	const getPluginStreams = (type: keyof TwtKprPluginConfiguration) =>
+		plugins.filter((plugin) => !!plugin[type]).map((plugin) => plugin[type]);
 
 	return async (req: Request, res: Response, next: NextFunction) => {
-		debug({ query: JSON.stringify(req.query) });
+		debug(`handling query`, JSON.stringify(req.query));
 
 		if (!Object.keys(req.query).length) {
 			next();
@@ -36,53 +40,119 @@ export default function queryHandler(
 		}
 
 		if (req.query[queryParameters.app] !== undefined) {
-			const appContent = renderApp({ mainRoute, uploadConfiguration });
+			debug('rendering app');
+			const appContent = renderApp({ mainRoute });
 			res.set('etag', generateEtag(appContent)).send(appContent);
 			return;
 		}
 
 		if (req.query[queryParameters.css] !== undefined) {
-			res.sendFile('styles.css', {
-				root: path.resolve(__dirname, 'client'),
+			debug('rendering css');
+			const mainStream = fs.createReadStream(
+				path.join(__dirname, 'client', 'styles.css')
+			);
+
+			const pluginStreams = getPluginStreams('clientCSS');
+
+			const streams = [mainStream, ...pluginStreams];
+
+			const combined = combineStreams(streams);
+			res.writeHead(200, {
+				'Content-Type': 'text/css',
 			});
+			combined.pipe(res);
+
+			combined.on('error', (error) => {
+				console.error('Error streaming file:', error);
+				res.end();
+			});
+
 			return;
 		}
 
 		if (req.query[queryParameters.js] !== undefined) {
-			res.sendFile('script.js', {
-				root: path.resolve(__dirname, 'client'),
+			debug('rendering js');
+			const mainStream = fs.createReadStream(
+				path.join(__dirname, 'client', 'script.js')
+			);
+
+			const pluginStreams = getPluginStreams('clientJS');
+			const streams = [mainStream, ...pluginStreams];
+
+			const combined = combineStreams(streams);
+			res.writeHead(200, {
+				'Content-Type': 'application/javascript',
 			});
+			combined.pipe(res);
+
+			combined.on('error', (error) => {
+				console.error('Error streaming file:', error);
+				res.end();
+			});
+
 			return;
 		}
 
 		if (
 			req.query[queryParameters.following] !== undefined &&
-			cache.get('following')
+			(await getFromCache('following'))
 		) {
-			return followingHandler(req, res, cache, queryParameters.following);
+			debug('rendering following');
+			const following = await getFromCache('following');
+			return followingHandler(
+				req,
+				res,
+				following as Twttr[],
+				queryParameters.following
+			);
 		}
 
 		if (
 			req.query[queryParameters.metadata] !== undefined &&
-			cache.get('metadata')
+			(await getFromCache('metadata'))
 		) {
-			return metadataHandler(req, res, cache, queryParameters.metadata);
+			debug('rendering metadata');
+			const metadata = (await getFromCache('metadata')) as Metadata;
+			return metadataHandler(req, res, metadata, queryParameters.metadata);
 		}
 
 		if (
 			(req.query[queryParameters.twt] !== undefined ||
 				req.query[queryParameters.twts] !== undefined) &&
-			cache.get('twts')
+			(await getFromCache('twts'))
 		) {
+			debug('rendering twts');
+			const twts = await getFromCache('twts');
 			return twtHandler(
 				req,
 				res,
-				cache.get('twts') as Twt[],
+				twts as Twt[],
 				queryParameters.twt,
 				queryParameters.twts
 			);
 		}
 
+		(plugins as TwtKprPluginConfiguration[]).forEach((plugin) => {
+			if (!plugin?.queryRoutes?.length) return;
+
+			plugin.queryRoutes.forEach(async (route) => {
+				// default to no auth
+				const { handler, queryParameter, requiresAuth = false } = route ?? {};
+
+				if (queryParameter && req.query[queryParameter] !== undefined) {
+					debug(`rendering plugin queryParameter ${queryParameter}`);
+
+					if (requiresAuth && !(await verifyAuthRequest(req))) {
+						debug('auth check failed');
+						next();
+						return;
+					}
+
+					return handler(req, res, next);
+				}
+			});
+		});
+
 		next();
 	};
 }

src/middlewares/queryHandler/twtHandler.ts

@@ -1,5 +1,5 @@
 import type { Request, Response } from 'express';
-import type { Metadata, Twt } from 'twtxt-lib';
+import type { Twt } from 'twtxt-lib';
 
 import {
 	generateEtag,
@@ -8,7 +8,6 @@ import {
 } from '../../lib/utils.js';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
-import NodeCache from '@cacheable/node-cache';
 import { QueryParameters } from '../../types.js';
 
 dayjs.extend(utc);