initial public commit

src/__tests__/hashTwt.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+
+import hashTwt from "../hashTwt.ts";
+
+describe("hashtwt", () => {
+	it("should correctly provide hash for known good twt", () => {
+		// https://twtxt.net/twt/g524g5q
+		const created = "2025-08-09T21:33:43-04:00";
+		const twt = {
+			content: "Is this thing on?",
+			created,
+			createdUTC: "",
+			url: "http://itsericwoodward.com/twtxt.txt",
+		};
+		const result = hashTwt(twt);
+		expect(result).toEqual("g524g5q");
+	});
+
+	it("should correctly provide hash for twt", () => {
+		const created = "2025-03-02T18:47:01+01:00";
+		// dayjs.utc(created).toISOString();
+		const twt = {
+			content:
+				"Normcore tilde ad selfies, culpa cupping nostrud gatekeep aesthetic PBR&B 3 wolf moon mustache twee.",
+			created,
+			createdUTC: "",
+			url: "https://example.com/demo-hipster-twtxt.txt",
+		};
+		const result = hashTwt(twt);
+		expect(result).toEqual("tvjursa");
+	});
+});

src/__tests__/parseTwtxt.mocks.ts

@@ -0,0 +1,105 @@
+export const hipsterMockData = {
+	following: [
+		{
+			nick: "demo_hipster",
+			url: "https://example.com/demo-hipster-twtxt.txt",
+		},
+		{
+			nick: "demo_pirate",
+			url: "https://example.org/~pirate/twtxt.txt",
+		},
+		{
+			nick: "demo_sagan",
+			url: "https://example.net/~saganos/twtxt.txt",
+		},
+	],
+	metadata: {
+		nick: "demo_hipster",
+		url: "https://example.com/demo-hipster-twtxt.txt",
+		avatar: "https://i.pravatar.cc/150?img=67",
+		description:
+			"Kitsch ut post-ironic, bruh tilde non shabby chic iceland fixie consequat?",
+	},
+	twts: [
+		{
+			content:
+				"Normcore tilde ad selfies, culpa cupping nostrud gatekeep aesthetic PBR&B 3 wolf moon mustache twee.",
+			created: "2025-06-02T18:47:01+01:00",
+			createdUTC: "2025-06-02T17:47:01.000Z",
+			hash: "ymiydvq",
+		},
+		{
+			content:
+				"Cardigan jean shorts eu 90's. Kitsch knausgaard culpa, marfa mumblecore portland raclette banjo retro exercitation pariatur snackwave williamsburg.",
+			created: "2025-07-05T00:17:46+02:00",
+			createdUTC: "2025-07-04T22:17:46.000Z",
+			hash: "c6bm4sq",
+		},
+		{
+			content:
+				"Literally deep v snackwave nostrud pug YOLO yes plz anim. JOMO crucifix bespoke chambray lomo keytar, labore ipsum.",
+			created: "2025-07-05T19:35:33+03:00",
+			createdUTC: "2025-07-05T16:35:33.000Z",
+			hash: "5yjtq4a",
+		},
+		{
+			content:
+				"Ut letterpress synth hoodie, wayfarers kitsch air plant eu selvage tilde taiyaki grailed cliche ex. Skateboard pariatur non leggings.",
+			created: "2025-08-05T22:46:11+04:00",
+			createdUTC: "2025-08-05T18:46:11.000Z",
+			hash: "gtotdwq",
+		},
+		{
+			content:
+				"Ad pug ex hashtag live-edge distillery affogato. Succulents hammock taiyaki biodiesel chartreuse, nulla you probably haven't heard of them four dollar toast quinoa keytar cornhole.",
+			created: "2025-08-08T13:49:20+05:00",
+			createdUTC: "2025-08-08T08:49:20.000Z",
+			hash: "2chryga",
+		},
+		{
+			content:
+				"Cardigan JOMO blackbird spyplane, whatever commodo pop-up normcore ad yr in eiusmod forage echo park exercitation +1.",
+			created: "2025-09-09T12:48:04+05:00",
+			createdUTC: "2025-09-09T07:48:04.000Z",
+			hash: "oqby2ja",
+		},
+		{
+			content:
+				"Culpa snackwave williamsburg, asymmetrical wolf microdosing literally. La croix coloring book jean shorts poutine, 3 wolf moon chicharrones hashtag chillwave affogato green juice.",
+			created: "2025-10-09T20:33:15+04:00",
+			createdUTC: "2025-10-09T16:33:15.000Z",
+			hash: "e4ylk3a",
+		},
+		{
+			content:
+				"Kickstarter kale chips williamsburg swag sunt disrupt chartreuse jianbing banh mi craft beer anim vaporware readymade.",
+			created: "2025-11-11T12:54:43+03:00",
+			createdUTC: "2025-11-11T09:54:43.000Z",
+			hash: "bdsl7tq",
+		},
+		{
+			content:
+				"Pinterest Brooklyn direct trade freegan. Health goth consequat bespoke ad hoodie in est ugh. IPhone typewriter lomo venmo. Hashtag chillwave hella lumbersexual in blackbird spyplane yr tbh. Yr waistcoat kogi est neutra hammock mollit. Drinking vinegar godard hell of occaecat direct trade. In 3 wolf moon jianbing bitters, roof party mixtape yuccie.",
+			created: "2025-12-05T17:15:28+02:00",
+			createdUTC: "2025-12-05T15:15:28.000Z",
+			hash: "qa45s7a",
+		},
+		{
+			content:
+				"Ethical twee swag, farm-to-table irure semiotics bodega boys umami sriracha stumptown cred four dollar toast tofu photo booth tbh.",
+			created: "2026-01-01T11:32:39+01:00",
+			createdUTC: "2026-01-01T10:32:39.000Z",
+			hash: "rbl3dxq",
+		},
+		{
+			content:
+				"(#4f5dlsa) <@demo_pirate https://example.org/~pirate/twtxt.txt> Asymmetrical kombucha trust fund jawn gentrify sartorial cloud bread artisan live-edge.",
+			created: "2026-02-01T13:13:13+02:00",
+			createdUTC: "2026-02-01T11:13:13.000Z",
+			hash: "drsaq7q",
+			replyHash: "4f5dlsa",
+			replyNick: "demo_pirate",
+			replyUrl: "https://example.org/~pirate/twtxt.txt",
+		},
+	],
+};

src/__tests__/parseTwtxt.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+import parseTwtxt from "../parseTwtxt.ts";
+import { hipsterMockData } from "./parseTwtxt.mocks.ts";
+
+export const loadFileFromSrc = async (...filePathFromSrc: string[]) => {
+	try {
+		return readFile(resolve(__dirname, "..", ...filePathFromSrc), "utf8");
+	} catch (error) {
+		console.error("Failed to read file: ", error);
+	}
+};
+
+describe("parseTwtxt", () => {
+	it("parses hipster file", async () => {
+		const fileText = await loadFileFromSrc(
+			"twtxt-demos",
+			"demo-hipster-twtxt.txt",
+		);
+		const fileData = parseTwtxt(fileText ?? "");
+		expect(fileData).toEqual(hipsterMockData);
+	});
+});

src/base32.js.d.ts

@@ -0,0 +1 @@
+declare module "base32.js";

src/constants.ts

@@ -0,0 +1,4 @@
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+export const __dirname = dirname(fileURLToPath(import.meta.url));

src/hashTwt.ts

@@ -0,0 +1,57 @@
+import { Buffer } from "buffer";
+globalThis.Buffer = Buffer;
+
+import type { Twt } from "./types.ts";
+
+import { blake2b } from "@exodus/blakejs";
+
+import { base32Encode } from "./utils.ts";
+
+const dateRegex =
+	/^(\d{4})-(\d{2})-(\d{2})([tT ])(\d{2}):(\d{2}):(\d{2})\.?(\d{3})?(?:(?:([+-]\d{2}):?(\d{2}))|Z)?$/;
+
+const formatRFC3339 = (date: string) => {
+	const pad = (num: number | string = 0) => `${+num < 10 ? 0 : ""}${+num}`;
+	const padYear = (num: number | string = 0) =>
+		`${+num < 1000 ? 0 : ""}${+num < 100 ? 0 : ""}${
+			+num < 10 ? 0 : ""
+		}${+num}`;
+
+	let m = dateRegex.exec(date);
+
+	//if timezone is undefined, it must be Z or nothing (otherwise the group would have captured).
+	if (m && m?.[9] === undefined) {
+		//Use UTC.
+		m[9] = "+00";
+	}
+	if (m && m?.[10] === undefined) {
+		m[10] = "00";
+	}
+
+	const offset = `${m?.[9]}:${m?.[10]}`.replace(/[+-]?00:00$/, "Z");
+
+	return [
+		padYear(m?.[1]),
+		"-",
+		pad(m?.[2]),
+		"-",
+		pad(m?.[3]),
+		m?.[4],
+		pad(m?.[5]),
+		":",
+		pad(m?.[6]),
+		":",
+		pad(m?.[7]),
+		//ignore milliseconds (m[8])
+		offset,
+	].join("");
+};
+
+export default function hashTwt(twt: Twt): string {
+	const created = formatRFC3339(twt.created);
+	const payload = [twt.url, created, twt.content].join("\n");
+
+	return base32Encode(blake2b(payload, undefined, 32))
+		.toLowerCase()
+		.slice(-7);
+}

src/index.ts

@@ -0,0 +1,6 @@
+export type * from "./types.ts";
+
+export { default as hashTwt } from "./hashTwt.ts";
+export { default as loadAndParseTwtxtFile } from "./loadAndParseTwtxt.ts";
+export { default as parseTwtxt } from "./parseTwtxt.ts";
+export { base32Encode } from "./utils.ts";

src/loadAndParseTwtxt.ts

@@ -0,0 +1,23 @@
+import dayjs from "dayjs";
+
+import parseTwtxt from "./parseTwtxt.js";
+
+export default async function loadAndParseTwtxtFile(url = "") {
+	if (!url) throw new Error("URL is required");
+
+	try {
+		const response = await fetch(url);
+		const twtxtFile = await response.text();
+		const lastModified = dayjs(
+			response.headers.get("Last-Modified"),
+		).toISOString();
+
+		return {
+			...parseTwtxt(twtxtFile),
+			lastModified,
+		};
+	} catch (err) {
+		console.error(err);
+		throw err;
+	}
+}

src/parseTwtxt.ts

@@ -0,0 +1,123 @@
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+
+import type { Metadata, Twttr } from "./types.ts";
+
+import hashTwt from "./hashTwt.js";
+import { getValueOrFirstEntry } from "./utils.ts";
+
+dayjs.extend(utc);
+
+/**
+ * @param twtxt
+ * @returns object containing: following, metadata, twts
+ */
+export default function parseTwtxt(twtxt: string) {
+	const allLines = twtxt.split("\n");
+
+	const { commentLines = [], contentLines = [] } = allLines.reduce(
+		(
+			acc: {
+				commentLines: string[];
+				contentLines: string[];
+			},
+			originalLine,
+		) => {
+			const line = originalLine.trim();
+			if (line === "") return acc;
+
+			if (line.startsWith("#")) acc.commentLines.push(line);
+			else acc.contentLines.push(line);
+			return acc;
+		},
+		{
+			commentLines: [],
+			contentLines: [],
+		},
+	);
+
+	const { following = [], metadata = {} } = commentLines
+		.filter((line) => line.includes("="))
+		.reduce(
+			(
+				acc: {
+					following: Twttr[];
+					metadata: Metadata;
+				},
+				line,
+			) => {
+				const [key, ...vals] = line
+					// remove #
+					.substring(1)
+					.split("=")
+					.map((field) => field.trim());
+				const val = vals.join("=");
+				if (key === "follow") {
+					const [nick, url] = val.trim().split(/\s+/);
+					acc.following.push({ nick, url });
+				} else {
+					if (acc.metadata[key]) {
+						if (!Array.isArray(acc.metadata[key]))
+							acc.metadata[key] = [acc.metadata[key], val];
+						else acc.metadata[key].push(val);
+					} else acc.metadata[key] = val;
+				}
+				return acc;
+			},
+			{
+				following: [],
+				metadata: {},
+			},
+		);
+
+	const replyRegEx = /\(#([\w]+)\) (\<\@(\S+) ([^>]+)>)*/;
+
+	const twts = contentLines
+		.map((line) => {
+			const [created, content] = line
+				.split(/\t/)
+				.map((val) => val.trim());
+
+			if (typeof content === "undefined")
+				throw new Error(`Content is undefined: ${line}`);
+
+			const createdDayjs = dayjs.utc(created);
+			if (!createdDayjs.isValid())
+				throw new Error(`Date is invalid: ${line}`);
+
+			const createdUTC = createdDayjs.toISOString();
+
+			const replyMatches = replyRegEx.exec(content);
+			let replyHash, replyNick, replyUrl;
+
+			if (replyMatches?.length) {
+				replyHash = replyMatches?.[1];
+				replyNick = replyMatches?.[3];
+				replyUrl = replyMatches?.[4];
+			}
+
+			const hash = hashTwt({
+				content,
+				created,
+				createdUTC,
+				url: getValueOrFirstEntry(metadata?.url ?? ""),
+			});
+
+			return {
+				content,
+				created,
+				createdUTC,
+				hash,
+				replyHash,
+				replyNick,
+				replyUrl,
+			};
+		})
+		.sort((a, b) => dayjs(a.created).diff(dayjs(b.created)));
+
+	return {
+		following,
+		metadata,
+		twts,
+	};
+}

src/twtxt-demos/demo-hipster-twtxt.txt

@@ -0,0 +1,20 @@
+# nick = demo_hipster
+# url = https://example.com/demo-hipster-twtxt.txt
+# avatar = https://i.pravatar.cc/150?img=67
+# description = Kitsch ut post-ironic, bruh tilde non shabby chic iceland fixie consequat?
+#
+# follow = demo_hipster https://example.com/demo-hipster-twtxt.txt
+# follow = demo_pirate https://example.org/~pirate/twtxt.txt
+# follow = demo_sagan https://example.net/~saganos/twtxt.txt
+
+2025-06-02T18:47:01+01:00	Normcore tilde ad selfies, culpa cupping nostrud gatekeep aesthetic PBR&B 3 wolf moon mustache twee.
+2025-07-05T00:17:46+02:00	Cardigan jean shorts eu 90's. Kitsch knausgaard culpa, marfa mumblecore portland raclette banjo retro exercitation pariatur snackwave williamsburg.
+2025-07-05T19:35:33+03:00	Literally deep v snackwave nostrud pug YOLO yes plz anim. JOMO crucifix bespoke chambray lomo keytar, labore ipsum.
+2025-08-05T22:46:11+04:00	Ut letterpress synth hoodie, wayfarers kitsch air plant eu selvage tilde taiyaki grailed cliche ex. Skateboard pariatur non leggings.
+2025-08-08T13:49:20+05:00	Ad pug ex hashtag live-edge distillery affogato. Succulents hammock taiyaki biodiesel chartreuse, nulla you probably haven't heard of them four dollar toast quinoa keytar cornhole.
+2025-09-09T12:48:04+05:00	Cardigan JOMO blackbird spyplane, whatever commodo pop-up normcore ad yr in eiusmod forage echo park exercitation +1.
+2025-10-09T20:33:15+04:00	Culpa snackwave williamsburg, asymmetrical wolf microdosing literally. La croix coloring book jean shorts poutine, 3 wolf moon chicharrones hashtag chillwave affogato green juice.
+2025-11-11T12:54:43+03:00	Kickstarter kale chips williamsburg swag sunt disrupt chartreuse jianbing banh mi craft beer anim vaporware readymade.
+2025-12-05T17:15:28+02:00	Pinterest Brooklyn direct trade freegan. Health goth consequat bespoke ad hoodie in est ugh. IPhone typewriter lomo venmo. Hashtag chillwave hella lumbersexual in blackbird spyplane yr tbh. Yr waistcoat kogi est neutra hammock mollit. Drinking vinegar godard hell of occaecat direct trade. In 3 wolf moon jianbing bitters, roof party mixtape yuccie.
+2026-01-01T11:32:39+01:00	Ethical twee swag, farm-to-table irure semiotics bodega boys umami sriracha stumptown cred four dollar toast tofu photo booth tbh.
+2026-02-01T13:13:13+02:00	(#4f5dlsa) <@demo_pirate https://example.org/~pirate/twtxt.txt> Asymmetrical kombucha trust fund jawn gentrify sartorial cloud bread artisan live-edge.

src/types.ts

@@ -0,0 +1,35 @@
+export interface LoadAndParseTwtxtWithCacheConfig {
+	cacheKeyPrefix: string;
+	onLoad?: (data: Twtxt) => void;
+	user?: Twttr;
+}
+
+export interface Metadata {
+	[key: string]: string | string[];
+}
+
+export interface Twt {
+	avatar?: string;
+	content: string;
+	created: string;
+	createdUTC: string;
+	hash?: string;
+	nick?: string;
+	noDom?: boolean;
+	replyHash?: string;
+	replyNick?: string;
+	replyUrl?: string;
+	url?: string;
+}
+
+export interface Twttr {
+	avatar?: string;
+	nick: string;
+	url: string;
+}
+
+export interface Twtxt {
+	following: Twttr[];
+	metadata: Metadata;
+	twts: Twt[];
+}

src/utils.ts

@@ -0,0 +1,9 @@
+import base32 from "base32.js";
+
+export const base32Encode = (payload: string | Uint8Array<ArrayBufferLike>) => {
+	const encoder = new base32.Encoder({ type: "rfc4648" });
+	return encoder.write(payload).finalize();
+};
+
+export const getValueOrFirstEntry = (value: unknown | unknown[]) =>
+	Array.isArray(value) && value.length ? value[0] : value;