skills/defuddle-web-content/references/source/src/extractors/chatgpt.ts

import { ConversationExtractor } from './_conversation';
import { ExtractorOptions } from './_base';
import { ConversationMessage, ConversationMetadata, Footnote } from '../types/extractors';
import { parseHTML, serializeHTML, escapeHtml } from '../utils/dom';

export class ChatGPTExtractor extends ConversationExtractor {
	private turns: NodeListOf<Element> | null;
	private footnotes: Footnote[];
	private footnoteCounter: number;
	private cachedMessages: ConversationMessage[] | null = null;

	constructor(document: Document, url: string, schemaOrgData?: any, options?: ExtractorOptions) {
		super(document, url, schemaOrgData, options);
		this.turns = document.querySelectorAll('[data-testid^="conversation-turn-"]');
		this.footnotes = [];
		this.footnoteCounter = 0;
	}

	canExtract(): boolean {
		return !!this.turns && this.turns.length > 0;
	}

	protected extractMessages(): ConversationMessage[] {
		if (this.cachedMessages) return this.cachedMessages;

		const messages: ConversationMessage[] = [];
		this.footnotes = [];
		this.footnoteCounter = 0;

		if (!this.turns) return messages;

		this.turns.forEach((turn) => {
			// Get the localized author text from the sr-only heading and clean it
			const authorElement = turn.querySelector('h4.sr-only, h5.sr-only, h6.sr-only');
			const authorText = authorElement?.textContent
				?.trim()
				?.replace(/:\s*$/, '') // Remove colon and any trailing whitespace
				|| '';

			// ChatGPT can split one assistant turn into multiple message elements
			const messageEls = Array.from(turn.querySelectorAll('[data-message-author-role]'))
				.filter(el => el.closest('[data-testid^="conversation-turn-"]') === turn);
			const messageEl = messageEls[0];

			const currentAuthorRole = messageEl?.getAttribute('data-message-author-role') || '';

			// Assistant turns may have content before and after the Thought section,
			// so merge all message content fragments for the turn.
			const contentHtmlParts = messageEls.flatMap(el => {
				const contentElements = this.getMessageContentElements(el);
				return contentElements.length > 0
					? contentElements.map(contentEl => serializeHTML(contentEl))
					: [serializeHTML(el)];
			});

			let messageContent = (contentHtmlParts.length > 0 ? contentHtmlParts : [serializeHTML(turn)]).join('\n');
			messageContent = messageContent.replace(/\u200B/g, '');

			// Remove specific elements from the message content
			const tempDiv = this.document.createElement('div');
			tempDiv.appendChild(parseHTML(this.document, messageContent));
			tempDiv.querySelectorAll('h4.sr-only, h5.sr-only, h6.sr-only').forEach(el => el.remove());
			messageContent = serializeHTML(tempDiv);

			// Process inline references using regex to find the containers
			// Look for spans containing citation links (a[target=_blank][rel=noopener]), replacing entire structure
			// Also capture optional preceding ZeroWidthSpace
			const citationPattern = /(&ZeroWidthSpace;)?(<span[^>]*?>\s*(?:<span[^>]*?>\s*)*<a(?=[^>]*?href="([^"]+)")(?=[^>]*?target="_blank")(?=[^>]*?rel="noopener")[^>]*?>[\s\S]*?<\/a>\s*(?:<\/span>\s*)+)/gi;

			messageContent = messageContent.replace(citationPattern, (match, zws, spanStructure, url) => {
				// url is captured group 3
				let domain = '';
				let fragmentText = '';

				try {
					// Extract domain without www.
					domain = new URL(url).hostname.replace(/^www\./, '');

					// Extract and decode the fragment text if it exists
					const hashParts = url.split('#:~:text=');
					if (hashParts.length > 1) {
						fragmentText = decodeURIComponent(hashParts[1]);
						fragmentText = fragmentText.replace(/%2C/g, ',');

						const parts = fragmentText.split(',');
						if (parts.length > 1 && parts[0].trim()) {
							fragmentText = ` — ${parts[0].trim()}...`;
						} else if (parts[0].trim()) {
							fragmentText = ` — ${fragmentText.trim()}`;
						} else {
							fragmentText = '';
						}
					}
				} catch (e) {
					console.error(`Failed to parse URL: ${url}`, e);
					domain = url;
				}

				// Check if this URL already exists in our footnotes
				let footnoteIndex = this.footnotes.findIndex(fn => fn.url === url);
				let footnoteNumber: number;

				if (footnoteIndex === -1) {
					this.footnoteCounter++;
					footnoteNumber = this.footnoteCounter;
					this.footnotes.push({
						url,
						text: `<a href="${escapeHtml(url)}">${escapeHtml(domain)}</a>${escapeHtml(fragmentText)}`
					});
				} else {
					footnoteNumber = footnoteIndex + 1;
				}

				// Return just the footnote reference, replacing the ZWS (if captured) and the entire span structure
				return `<sup id="fnref:${footnoteNumber}"><a href="#fn:${footnoteNumber}">${footnoteNumber}</a></sup>`;
			});

			const cleanupDiv = this.document.createElement('div');
			cleanupDiv.appendChild(parseHTML(this.document, messageContent));
			cleanupDiv.querySelectorAll('span[data-state="closed"]').forEach(el => el.remove());
			messageContent = serializeHTML(cleanupDiv);

			// Clean up any stray empty paragraph tags
			messageContent = messageContent
				.replace(/<p[^>]*>\s*<\/p>/g, '');

			messages.push({
				author: authorText,
				content: messageContent.trim(),
				metadata: {
					role: currentAuthorRole || 'unknown'
				}
			});

		});

		this.cachedMessages = messages;
		return messages;
	}

	private getMessageContentElements(messageEl: Element): Element[] {
		const contentSelector = '.markdown, .whitespace-pre-wrap';
		const candidates = [
			...(messageEl.matches(contentSelector) ? [messageEl] : []),
			...Array.from(messageEl.querySelectorAll(contentSelector))
		];

		return candidates.filter(candidate =>
			!candidates.some(other => other !== candidate && other.contains(candidate))
		);
	}

	protected getFootnotes(): Footnote[] {
		return this.footnotes;
	}

	protected getMetadata(): ConversationMetadata {
		const title = this.getTitle();
		const messages = this.extractMessages();

		return {
			title,
			site: 'ChatGPT',
			url: this.url,
			messageCount: messages.length,
			description: `ChatGPT conversation with ${messages.length} messages`
		};
	}

	private getTitle(): string {
		// Try to get the page title first
		const pageTitle = this.document.title?.trim();
		if (pageTitle && pageTitle !== 'ChatGPT') {
			return pageTitle;
		}

		// Fall back to first user message
		const firstUserTurn = this.turns?.item(0)?.querySelector('.text-message');
		if (firstUserTurn) {
			const text = firstUserTurn.textContent || '';
			// Truncate to first 50 characters if longer
			return text.length > 50 ? text.slice(0, 50) + '...' : text;
		}

		return 'ChatGPT Conversation';
	}
}