JSFiddle - React, Tailwind, and code Playground
JavaScript
string = "This這 is是 English中文 …";
function byteCount(text) {
return encodeURI(text).split(/%..|./).length - 1;
}
function tokenize(text, targetLen) {
var result = [];
var pos = 0;
var current = "";
while (pos < text.length) {
var next = current + text.charAt(pos);
if (byteCount(next) > targetLen) {
result.push(current);
current = "";
pos--;
}
else if (byteCount(next) == targetLen) {
result.push(next);
current = "";
}
else {
current = next;
}
pos++;
}
if (current != "") {
result.push(current);
}
return result;
};
console.log(tokenize(string, 10));