srctree

Robin Linden parent b8d4ea5f 819ac74d
css2: Support negative integers and the comment-delimiter-close token

inlinesplit
css2/tokenizer.cpp added: 54, removed: 11, total 43
@@ -33,12 +33,6 @@ void Tokenizer::run() {
return;
}
 
if (inputs_starts_ident_sequence(*c)) {
temporary_buffer_ = *c;
state_ = State::IdentLike;
continue;
}
 
switch (*c) {
case ' ':
case '\n':
@@ -76,6 +70,28 @@ void Tokenizer::run() {
case ',':
emit(CommaToken{});
continue;
case '-': {
// TODO(robinlinden): This only handles integers.
if (auto next_input = peek_input(0); next_input && util::is_digit(*next_input)) {
auto number = consume_number(*c);
emit(NumberToken{number.second, number.first});
continue;
}
 
if (peek_input(0) == '-' && peek_input(1) == '>') {
emit(CdcToken{});
pos_ += 2;
continue;
}
 
if (inputs_starts_ident_sequence(*c)) {
reconsume_in(State::IdentLike);
continue;
}
 
emit(DelimToken{'-'});
continue;
}
case ':':
emit(ColonToken{});
continue;
@@ -110,9 +126,17 @@ void Tokenizer::run() {
continue;
}
default:
emit(DelimToken{*c});
continue;
break;
}
 
if (inputs_starts_ident_sequence(*c)) {
temporary_buffer_ = *c;
state_ = State::IdentLike;
continue;
}
 
emit(DelimToken{*c});
continue;
}
 
case State::CommentStart: {
@@ -330,7 +354,10 @@ std::pair<std::variant<int, double>, NumericType> Tokenizer::consume_number(char
std::string repr{};
 
std::optional<char> next_input;
if (first_byte == '+') {
if (first_byte == '-') {
repr += first_byte;
next_input = consume_next_input_character();
} else if (first_byte == '+') {
next_input = consume_next_input_character();
} else {
next_input = first_byte;
 
css2/tokenizer_test.cpp added: 54, removed: 11, total 43
@@ -332,5 +332,21 @@ int main() {
expect_token(output, IdentToken{"hello"});
});
 
etest::test("hyphen: negative number", [] {
auto output = run_tokenizer("-13");
expect_token(output, NumberToken{.data = -13});
});
 
etest::test("hyphen: negative number w/ leading 0", [] {
auto output = run_tokenizer("-00000001");
expect_token(output, NumberToken{.data = -1});
});
 
etest::test("hyphen: cdc", [] {
auto output = run_tokenizer("-->lol");
expect_token(output, CdcToken{});
expect_token(output, IdentToken{"lol"});
});
 
return etest::run_all_tests();
}