srctree

Robin Linden parent 8f610c6d 01abf619
js: Support tokenizing int literals

inline split

js/tokenizer.h added: 39, removed: 2, total 37

@@ -15,6 +15,11 @@

namespace js::parse {

struct IntLiteral {

int value{};

bool operator==(IntLiteral const &) const = default;

};

struct Identifier {

std::string name;

bool operator==(Identifier const &) const = default;

@@ -37,6 +42,7 @@ struct Eof {

};

using Token = std::variant< //

IntLiteral,

Identifier,

LParen,

RParen,

@@ -67,6 +73,10 @@ public:

break;

}

if (is_numeric(current)) {

return tokenize_int_literal(current);

}

assert(is_alpha(current));

return tokenize_identifier(current);

}

@@ -82,6 +92,22 @@ private:

return std::nullopt;

}

Token tokenize_int_literal(char current) {

int value{};

while (true) {

value += current - '0';

auto next = peek();

if (!next || !is_numeric(*next)) {

break;

}

value *= 10;

current = *next;

pos_ += 1;

}

return IntLiteral{value};

}

Token tokenize_identifier(char current) {

Identifier id{};

while (true) {

@@ -98,6 +124,7 @@ private:

}

static constexpr bool is_alpha(char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); }

static constexpr bool is_numeric(char c) { return (c >= '0' && c <= '9'); }

static constexpr bool is_whitespace(char c) {

switch (c) {

case ' ':

js/tokenizer_test.cpp added: 39, removed: 2, total 37

@@ -15,6 +15,11 @@ using etest::expect_eq;

using Tokens = std::vector<Token>;

int main() {

etest::test("int literal", [] {

expect_eq(tokenize("13"), Tokens{IntLiteral{13}, Eof{}});

expect_eq(tokenize("0"), Tokens{IntLiteral{0}, Eof{}});

});

etest::test("identifier", [] {

expect_eq(tokenize("hello"), Tokens{Identifier{"hello"}, Eof{}}); //

});

@@ -31,5 +36,10 @@ int main() {

expect_eq(tokenize("func ( ) ;"), Tokens{Identifier{"func"}, LParen{}, RParen{}, Semicolon{}, Eof{}}); //

});

etest::test("function call w/ numeric argument", [] {

expect_eq(tokenize("func(9)"), Tokens{Identifier{"func"}, LParen{}, IntLiteral{9}, RParen{}, Eof{}});

expect_eq(tokenize("func( 9 )"), Tokens{Identifier{"func"}, LParen{}, IntLiteral{9}, RParen{}, Eof{}});

});

return etest::run_all_tests();

}