SentenceStream

by Matt

JavaScript

main()


async function main() {

	console.log('hello!')
  const text = `Hey there! What's up... All good?`

  const textStream = iteratorToStream([...text].values())
  const sentenceStream = textStream.pipeThrough(new TextToSentenceStream())
  const iterator = streamAsyncIterator(sentenceStream)
  
  for await (const sentence of iterator) {
  	console.log('sentence:', sentence)
  } 

}


class TextToSentenceStream extends TransformStream {
	constructor() {
		let buffer = "";

		function detectSentences(buffer) {
			const pattern = /[^\s][^.!?:]*[.!?:]+(?=[^.!?:])/g;

			// const match = pattern.exec(buffer)
			// match!.index

			let match;
			let lastIndex = 0;
			const sentences = [];

			while ((match = pattern.exec(buffer))) {
				sentences.push(match[0]);
				lastIndex = pattern.lastIndex;
			}

			return {
				sentences,
				lastIndex,
			};
		}

		/**
		 * Only enqueue sentences that contain words, ignore dangling punctuation
		 */
		function enqueueNonEmpty(
			chunk,
			controller
		) {
			if (chunk.match(/\w/)) {
				const trimmed = chunk.trim(); // trim whitespace
				controller.enqueue(trimmed);
			}
		}

		super({
			transform(chunk, controller) {
				buffer += chunk;

				const { sentences, lastIndex } = detectSentences(buffer);

				for (const sentence in sentences) {
					enqueueNonEmpty(sentence, controller);
				}

				buffer = buffer.slice(lastIndex);
			},
			flush(controller) {
				enqueueNonEmpty(buffer, controller);
			},
		});
	}
}

async function* streamAsyncIterator(stream) {
	// Get a lock on the stream
	const reader = stream.getReader();

	try {
		while (true) {
			// Read from the stream
			const { done, value } = await reader.read();
			// Exit if we're done
			if (done) return;
			// Else yield the chunk
			yield value;
		}
	} finally {
		reader.releaseLock();
	}
}

function iteratorToStream(iterator) {
	return new ReadableStream({
		async pull(controller) {
			const { value, done } = await iterator.next();

			if (done)...