String aware tokenizer

A "string" aware string tokenizer

by justjohn

JavaScript

function tokenizeString(command) {
     var tokens = []
        , token = []
         , i = 0
         , chr
        , split = " "
        , string = null
        , ignore = false;
         
     for (;i < command.length; i++) {
         chr = command.charAt(i);
        
        if (!ignore) {
            switch (chr) {
                case "\\":
                    ignore = true;
                    break;
                        
                case '"':
                case "'":
                    if (!string) {
                        // Start a string
                        string = chr;
                    } else if (string === chr) {
                        // End a string
                        string = null;
                    }
                    break;
                        
                case split:
                    if (!string) {
                        tokens.push(token.join("").trim());
                        token = [];
                    }
                    break;
            }
        } else {
            ignore = false;
        }
        token.push(chr);
    }
    // push the last token to the stack
    if (token.length > 0) tokens.push(token.join("").trim());
     
    if (string != null) throw "Unclosed string";
     
    return tokens;
}

tokens = tokenizeString('ls -la "/Users/\\\"john"');
console.log(tokens);