srctree

Robin Linden parent 7f18cc70 4a342b6e
js: Ignore whitespace when tokenizing

inlinesplit
js/tokenizer.h added: 30, removed: 6, total 24
@@ -48,11 +48,14 @@ public:
explicit Tokenizer(std::string_view input) : input_{input} {}
 
Token tokenize() {
if (pos_ >= input_.size()) {
return Eof{};
}
char current{};
do {
if (pos_ >= input_.size()) {
return Eof{};
}
current = input_[pos_++];
} while (is_whitespace(current));
 
char current = input_[pos_++];
switch (current) {
case '(':
return LParen{};
@@ -91,6 +94,19 @@ private:
}
 
static constexpr bool is_alpha(char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); }
static constexpr bool is_whitespace(char c) {
switch (c) {
case ' ':
case '\n':
case '\r':
case '\f':
case '\v':
case '\t':
return true;
default:
return false;
}
}
};
 
inline std::vector<Token> tokenize(std::string_view input) {
 
js/tokenizer_test.cpp added: 30, removed: 6, total 24
@@ -19,9 +19,17 @@ int main() {
expect_eq(tokenize("hello"), Tokens{Identifier{"hello"}, Eof{}}); //
});
 
etest::test("identifiers w/ whitespace", [] {
expect_eq(tokenize(" lol no "), Tokens{Identifier{"lol"}, Identifier{"no"}, Eof{}}); //
});
 
etest::test("function call", [] {
expect_eq(tokenize("func();"), Tokens{Identifier{"func"}, LParen{}, RParen{}, Semicolon{}, Eof{}}); //
});
 
etest::test("function call w/ whitespace", [] {
expect_eq(tokenize("func ( ) ;"), Tokens{Identifier{"func"}, LParen{}, RParen{}, Semicolon{}, Eof{}}); //
});
 
return etest::run_all_tests();
}