JSFiddle - React, Tailwind, and code Playground

by Matt

HTML

<script type="importmap">
	{
		"imports": {
			"soundswallower": "https://cdn.jsdelivr.net/npm/[email protected]/+esm"
		}
	}
</script>

CSS

canvas {
	position: fixed;
	inset: 0;
}

JavaScript

import soundswallower_factory from "soundswallower";

async function main() {
	console.log("initializing...")
	const recognizer = new SoundSwallowerRecognizer()
  await recognizer.initialize()
  
  console.log("initialized")
}

main()

const SoundSwallowerPhonemes = [
	"+NSN+",
	"+SPN+",
	"SIL",
	"AA",
	"AE",
	"AH",
	"AO",
	"AW",
	"AY",
	"B",
	"CH",
	"D",
	"DH",
	"EH",
	"ER",
	"EY",
	"F",
	"G",
	"HH",
	"IH",
	"IY",
	"JH",
	"K",
	"L",
	"M",
	"N",
	"NG",
	"OW",
	"OY",
	"P",
	"R",
	"S",
	"SH",
	"T",
	"TH",
	"UH",
	"UW",
	"V",
	"W",
	"Y",
	"Z",
	"ZH",
]

class SoundSwallowerPhonemeRecognizer {

	async initialize(config) {
		this.soundswallower = await soundswallower_factory({
			modelBase: "https://cdn.jsdelivr.net/npm/[email protected]/model/",
			locateFile(url) {
				if (url === "soundswallower.web.wasm") {
					return "https://cdn.jsdelivr.net/npm/[email protected]/soundswallower.web.wasm";
				}
				return url;
			},
		})

		const recognizer = new this.soundswallower.Decoder(config);
		await recognizer.initialize();

		// Setup phonemes dict and grammar

		const nonSilentPhonemes = SoundSwallowerPhonemes.filter(
			(phone) => !["+NSN+", "+SPN+", "SIL"].includes(phone)
		);
		const dictEntries = nonSilentPhonemes.map((phone) => [
			phone,
			phone,
		]);
		recognizer.add_words(...dictEntries);

		const grammar = `
#JSGF V1.0;
grammar phones;
public <speech> = ([<phone>])*;
<phone> = ${nonSilentPhonemes.join(" | ")};
		`;
		recognizer.set_grammar(grammar);

		this.recognizer = recognizer;
	}

	async recognize(data, sampleRate) {
		const recognizer = this.recognizer;
		if (!recognizer) throw new Error("recognizer not initialized");

		if (recognizer.get_config("samprate") != sampleRate) {
			recognizer.set_config("samprate", sampleRate);
			await recognizer.reinitialize_audio();
		}

		recognizer.start();
		recognizer.process_audio(data, false, true);
		recognizer.stop();

		const speech = recognizer.get_alignment({
			align_level: 1,
		});

		return...