srctree

Mikael Larsson parent 5bc3e224 8c168b9e
css2: Add support for tokenizing single code point tokens

inlinesplit
css2/tokenizer.cpp added: 85, removed: 4, total 81
@@ -57,6 +57,33 @@ void Tokenizer::run() {
case '-':
state_ = State::HyphenMinus;
continue;
case '(':
emit(OpenParenToken{});
continue;
case ')':
emit(CloseParenToken{});
continue;
case ',':
emit(CommaToken{});
continue;
case ':':
emit(ColonToken{});
continue;
case ';':
emit(SemiColonToken{});
continue;
case '[':
emit(OpenSquareToken{});
continue;
case ']':
emit(CloseSquareToken{});
continue;
case '{':
emit(OpenCurlyToken{});
continue;
case '}':
emit(CloseCurlyToken{});
continue;
default:
emit(DelimToken{*c});
continue;
 
css2/tokenizer_test.cpp added: 85, removed: 4, total 81
@@ -84,9 +84,9 @@ int main() {
});
 
etest::test("delimiter after comment", [] {
auto output = run_tokenizer("/*/*/,");
auto output = run_tokenizer("/*/*/?");
 
expect_token(output, DelimToken{','});
expect_token(output, DelimToken{'?'});
});
 
etest::test("eof in comment", [] {
@@ -200,5 +200,59 @@ int main() {
expect_token(output, WhitespaceToken{});
});
 
etest::test("open paren", [] {
auto output = run_tokenizer("(");
 
expect_token(output, OpenParenToken{});
});
 
etest::test("close paren", [] {
auto output = run_tokenizer(")");
 
expect_token(output, CloseParenToken{});
});
 
etest::test("comma", [] {
auto output = run_tokenizer(",");
 
expect_token(output, CommaToken{});
});
 
etest::test("colon", [] {
auto output = run_tokenizer(":");
 
expect_token(output, ColonToken{});
});
 
etest::test("semicolon", [] {
auto output = run_tokenizer(";");
 
expect_token(output, SemiColonToken{});
});
 
etest::test("open square bracket", [] {
auto output = run_tokenizer("[");
 
expect_token(output, OpenSquareToken{});
});
 
etest::test("close square bracket", [] {
auto output = run_tokenizer("]");
 
expect_token(output, CloseSquareToken{});
});
 
etest::test("open curly bracket", [] {
auto output = run_tokenizer("{");
 
expect_token(output, OpenCurlyToken{});
});
 
etest::test("close curly bracket", [] {
auto output = run_tokenizer("}");
 
expect_token(output, CloseCurlyToken{});
});
 
return etest::run_all_tests();
}