Word Segmenter

by wio_dude

HTML

<div>
  <textarea id="input" cols="70" rows="20"></textarea>
  <textarea id="output" cols="20" rows="20" readonly></textarea>
  <br/>
  <input type="button" id="transfer" value="Transfer" />
</div>

<div>
  <textarea id="segments" cols="20" rows="50"></textarea>
  <textarea id="words" cols="20" rows="50" readonly></textarea>
</div>

JavaScript

const wordSegmenter = new Intl.Segmenter(['zh-Hans-CN'], {
  granularity: 'word'
});

const countStartList = [
  '这',
  '那',
  '哪',
  '零',
  '一',
  '二',
  '两',
  '三',
  '四',
  '五',
  '六',
  '七',
  '八',
  '九',
  '十',
  '百',
  '千',
  '万',
];

const counterList = [
  '个',
  '只',
  '天',
];

const weirdStarts = [
  '很',
  '太',
];

const weirdEnds = [
  '上',
  '下',
  '里',
  '了',
  '的',
  '看',
]

const byId = (id) => document.getElementById(id);
const $e = {};

$e.input = byId('input');
$e.output = byId('output');

$e.submit = byId('submit');
$e.transfer = byId('transfer');

$e.segments = byId('segments');
$e.words = byId('words');

$e.unique = byId('unique');


$e.input.addEventListener('input', (_evt) => {
  $e.output.value = segmentText($e.input.value);
});

$e.transfer.addEventListener('click', (_evt) => {
  $e.segments.value = output.value;
  $e.words.value = removeDuplicates($e.segments.value);
});

$e.segments.addEventListener('input', (_evt) => {
  $e.words.value = removeDuplicates($e.segments.value);
});

function generateWordList(text) {
  const segmentedText = segmentWords(text);
  const uniqueSegments = dedup(segments);
}



function segmentWords(text) {
  const segmentedText = wordSegmenter.segment(text);
  return [...segmentedText]
    .filter(s => s.isWordLike)
    .map(s => s.segment);
}

function dedup(words) {
  const unique = new Set();
  for (const word of words) {
    if (!unique.has(word)) {
      unique.add(word);
      result.push(word);
    }
  }
  return result;
}


function segmentText(text) {
  const segmentedText = wordSegmenter.segment(text);
  return [...segmentedText].filter(s => s.isWordLike)
    .map(s => s.segment)
    .join('\n');
}

function removeDuplicates(text) {
  const result = [];
  const unique = new Set();
  const words = text.split('\n');
  for (const word of words) {
    if (!unique.has(word)) {
      unique.add(word);
      result.push(word);
    }
  }
  return result.join('\n');
}