diff --git a/src/lexer.zig b/src/lexer.zig index 00eeb1d..67e310b 100644 --- a/src/lexer.zig +++ b/src/lexer.zig @@ -1,4 +1,4 @@ -///! Lexer for `nf` file format to tokenize input sources accordningly. +///! Lexer for `smd` file format to tokenize input sources accordningly. pub const Token = struct { tag: Tag, loc: Location, @@ -9,48 +9,26 @@ pub const Token = struct { }; pub const Tag = enum(u8) { - anchor, - at_sign, + asterik, block, - colon, - comment, eof, - equal, hashtag, + heading, + indentation, invalid, - l_angle_bracket, - r_angle_bracket, - l_bracket, - l_bracket_colon, - l_bracket_minus, - l_bracket_slash, - r_bracket, + link, minus, - newline, - reference, - pipe, - pipe_equal, - plus, + separator, text, + tick, underscore, + quote, pub fn lexeme(tag: Tag) ?[]const u8 { return switch (tag) { - .at_sign => "@", - .colon => ":", - .equal => "=", - .hashtag => "#", - .l_angle_bracket => "<", - .r_angle_bracket => ">", - .l_bracket => "[", - .l_bracket_colon => "[:", - .l_bracket_minus => "[-", - .l_bracket_slash => "[/", - .r_bracket => "]", + .asterik => "*", .minus => "-", - .pipe => "|", - .pipe_equal => "|=", - .plus => "+", + .tick => "`", .underscore => "_", else => null, }; @@ -58,14 +36,15 @@ pub const Token = struct { pub fn symbol(tag: Tag) []const u8 { return tag.lexeme() orelse switch (tag) { - .anchor => "anchor", .block => "block", - .comment => "comment", .eof => "EOF", + .hashtag => "#hashtag", + .heading => "heading", + .indentation => "indentation", .invalid => "invalid token", - .newline => "\n", - .reference => "reference", + .separator => "---", .text => "text", + .quote => "quote", else => unreachable, }; } @@ -77,7 +56,7 @@ pub const Tokenizer = struct { index: u32, /// For debugging purposes - pub fn dump(self: *Tokenizer, token: *const Token) void { + pub fn dump(self: *const Tokenizer, token: *const Token) void { assert(token.loc.start < token.loc.end); print(".{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] }); } @@ -93,14 +72,15 @@ pub const Tokenizer = struct { const State = enum { default, invalid, - at_sign, - equal, - l_angle_bracket, - comment, + hashtag, + tag, + heading, + minus, + tick, block, - slash, - pipe, + r_angle_bracket, l_bracket, + l_brace, }; /// state fsm (finite state machine) describing the syntax of `nf` @@ -116,6 +96,28 @@ pub const Tokenizer = struct { /// -> do not group tokens, instead this should be done by the parser when deriving the ast from the token stream /// then the parser can identify missing parts and even point to the corresponding location in the file! pub fn next(this: *Tokenizer) Token { + const token = this.next_token(); + this.index = token.loc.end; + return token; + } + + /// Peek the next `Token` which would be seen by the Tokenizer after n calls to `next` without changing the internal state of the iterator. + /// This allows look ahead parsing of the Token stream. + pub fn peek(this: *Tokenizer, n: u32) Token { + assert(n > 0); + const start = this.index; + defer this.index = start; + + var token: Token = undefined; + for (0..n) |_| { + token = this.next_token(); + this.index = token.loc.end; + } + return token; + } + + fn next_token(this: *const Tokenizer) Token { + var index = this.index; var result: Token = .{ .tag = undefined, .loc = .{ @@ -124,121 +126,68 @@ pub const Tokenizer = struct { }, }; state: switch (State.default) { - .default => switch (this.buffer[this.index]) { - 0 => if (this.index == this.buffer.len) { - if (result.loc.start != this.index) { + .default => switch (this.buffer[index]) { + 0 => if (index == this.buffer.len) { + if (result.loc.start != index) { result.tag = .text; } else { return .{ .tag = .eof, .loc = .{ - .start = this.index, - .end = this.index, + .start = index, + .end = index, }, }; } } else { continue :state .invalid; }, - '=' => if (result.loc.start != this.index) { + '_' => if (result.loc.start != index) { result.tag = .text; } else { - this.index += 1; - result.tag = .equal; - }, - '@' => if (result.loc.start != this.index) { - result.tag = .text; - } else { - continue :state .at_sign; - }, - '|' => if (result.loc.start != this.index) { - result.tag = .text; - } else { - continue :state .pipe; - }, - '/' => continue :state .slash, - '`' => { - var i: u32 = 1; - while (this.buffer[this.index + i] != '`') : (i += 1) { - if (this.index + i >= this.buffer.len) { - this.index += 1; - continue :state .default; - } - } else { - this.index += i; - continue :state .default; - } - }, - '_' => if (result.loc.start != this.index) { - result.tag = .text; - } else { - this.index += 1; + index += 1; result.tag = .underscore; }, - '#' => if (result.loc.start != this.index) { + '*' => if (result.loc.start != index) { result.tag = .text; } else { - this.index += 1; - switch (this.buffer[this.index]) { - '#' => continue :state .block, - else => continue :state .default, - } + index += 1; + result.tag = .asterik; }, - ':' => if (result.loc.start != this.index) { + '#' => if (result.loc.start != index) { result.tag = .text; } else { - this.index += 1; - result.tag = .colon; + continue :state .hashtag; }, - '[' => if (result.loc.start != this.index) { + '>' => if (result.loc.start != index) { + result.tag = .text; + } else { + continue :state .r_angle_bracket; + }, + '[' => if (result.loc.start != index) { result.tag = .text; } else { continue :state .l_bracket; }, - ']' => if (result.loc.start != this.index) { + '-' => if (result.loc.start != index) { result.tag = .text; } else { - this.index += 1; - result.tag = .r_bracket; + continue :state .minus; }, - '<' => if (result.loc.start != this.index) { + '`' => if (result.loc.start != index) { result.tag = .text; } else { - continue :state .l_angle_bracket; - }, - '>' => if (result.loc.start != this.index) { - result.tag = .text; - } else { - this.index += 1; - result.tag = .r_angle_bracket; - }, - '+' => if (result.loc.start != this.index) { - result.tag = .text; - } else { - this.index += 1; - result.tag = .plus; - }, - '-' => if (result.loc.start != this.index) { - result.tag = .text; - } else { - this.index += 1; - result.tag = .minus; - }, - '\n' => if (result.loc.start != this.index) { - result.tag = .text; - } else { - this.index += 1; - result.tag = .newline; + continue :state .tick; }, else => { - this.index += 1; + index += 1; continue :state .default; }, }, .invalid => { - this.index += 1; - switch (this.buffer[this.index]) { - 0 => if (this.index == this.buffer.len) { + index += 1; + switch (this.buffer[index]) { + 0 => if (index == this.buffer.len) { result.tag = .invalid; } else { continue :state .invalid; @@ -246,81 +195,64 @@ pub const Tokenizer = struct { else => continue :state .invalid, } }, - // referencing - .at_sign => { - this.index += 1; - switch (this.buffer[this.index]) { - 'a'...'z', 'A'...'Z', '0'...'9', '-', '.', ':', '/' => continue :state .at_sign, - ' ', '\n', '\t', '\r', ']', '|', 0 => result.tag = .reference, - else => continue :state .invalid, - } - }, - .l_angle_bracket => { - this.index += 1; - switch (this.buffer[this.index]) { - 'a'...'z', 'A'...'Z', '0'...'9', '-', '.', ':', '/' => continue :state .l_angle_bracket, - '>' => { - this.index += 1; - result.tag = .anchor; + .r_angle_bracket => { + index += 1; + switch (this.buffer[index]) { + '\n' => { + // what about indentation? it should match the current 'level' + if (this.buffer[index + 1] == '>') { + index += 1; + continue :state .r_angle_bracket; + } + result.tag = .quote; }, - else => continue :state .invalid, + 0 => result.tag = .quote, + else => continue :state .r_angle_bracket, } }, - // links .l_bracket => { - this.index += 1; - switch (this.buffer[this.index]) { - ':' => { - this.index += 1; - result.tag = .l_bracket_colon; - }, - '-' => { - this.index += 1; - result.tag = .l_bracket_minus; - }, - '/' => { - this.index += 1; - result.tag = .l_bracket_slash; - }, - else => { - this.index -= 1; - result.tag = .l_bracket; + index += 1; + switch (this.buffer[index]) { + ']' => { + index += 1; + switch (this.buffer[index]) { + '(' => continue :state .l_brace, + else => result.tag = .link, + } }, + 0, '\n' => result.tag = .invalid, + else => continue :state .l_bracket, } }, - .pipe => { - this.index += 1; - switch (this.buffer[this.index]) { - '=' => { - this.index += 1; - result.tag = .pipe_equal; + .l_brace => { + index += 1; + switch (this.buffer[index]) { + ')' => { + index += 1; + result.tag = .link; }, - else => result.tag = .pipe, + 0, '\n' => result.tag = .invalid, + else => continue :state .l_brace, } }, - .slash => { - this.index += 1; - switch (this.buffer[this.index]) { - '/' => if (result.loc.start != this.index - 1) { - result.tag = .text; - this.index -= 1; - } else continue :state .comment, - else => continue :state .default, - } - }, - .comment => { - this.index += 1; - switch (this.buffer[this.index]) { - 0, '\n' => result.tag = .comment, - else => continue :state .comment, + .tick => { + index += 1; + switch (this.buffer[index]) { + '`' => if (this.buffer[index + 1] == '`') { + index += 2; + continue :state .block; + } else { + result.tag = .tick; + }, + else => result.tag = .tick, } }, .block => { - this.index += 1; - switch (this.buffer[this.index]) { + index += 1; + switch (this.buffer[index]) { 0 => result.tag = .invalid, - '#' => if (this.buffer[this.index + 1] == '#') { - this.index += 2; + '`' => if (this.buffer[index + 1] == '`' and this.buffer[index + 2] == '`') { + index += 3; result.tag = .block; } else { continue :state .block; @@ -328,241 +260,45 @@ pub const Tokenizer = struct { else => continue :state .block, } }, - else => { - print("Not yet implemented at {d}: '{s}'", .{ this.index, this.buffer[result.loc.start..this.index] }); - unreachable; - }, // not yet implemented - } - - result.loc.end = this.index; - return result; - } - - /// Peek the next `Token` which would be seen by the Tokenizer after n calls to `next` without changing the internal state of the iterator. - /// This allows look ahead parsing of the Token stream. - pub fn peek(this: Tokenizer, n: u32) Token { - assert(n > 0); - var index = this.index; - var result: Token = undefined; - for (0..n) |_| { - result = .{ - .tag = undefined, - .loc = .{ - .start = index, - .end = undefined, - }, - }; - state: switch (State.default) { - .default => switch (this.buffer[index]) { - 0 => if (index == this.buffer.len) { - if (result.loc.start != index) { - result.tag = .text; - } else { - return .{ - .tag = .eof, - .loc = .{ - .start = index, - .end = index, - }, - }; - } + .hashtag => { + index += 1; + switch (this.buffer[index]) { + '#', ' ' => continue :state .heading, + else => continue :state .tag, + } + }, + .tag => { + index += 1; + switch (this.buffer[index]) { + ' ', 0, '\n' => result.tag = .hashtag, + else => continue :state .tag, + } + }, + .heading => { + index += 1; + switch (this.buffer[index]) { + 0, '\n' => result.tag = .heading, + else => continue :state .heading, + } + }, + .minus => { + index += 1; + switch (this.buffer[index]) { + '-' => if (this.buffer[index + 1] == '-') { + index += 2; + result.tag = .separator; } else { - continue :state .invalid; - }, - '=' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - result.tag = .equal; - }, - '@' => if (result.loc.start != index) { - result.tag = .text; - } else { - continue :state .at_sign; - }, - '|' => if (result.loc.start != index) { - result.tag = .text; - } else { - continue :state .pipe; - }, - '/' => continue :state .slash, - '`' => { - var i: u32 = 1; - while (this.buffer[index + i] != '`') : (i += 1) { - if (index + i >= this.buffer.len) { - index += 1; - continue :state .default; - } - } else { - index += i; - continue :state .default; - } - }, - '_' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - result.tag = .underscore; - }, - '#' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - switch (this.buffer[index]) { - '#' => continue :state .block, - else => continue :state .default, - } - }, - ':' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - result.tag = .colon; - }, - '[' => if (result.loc.start != index) { - result.tag = .text; - } else { - continue :state .l_bracket; - }, - ']' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - result.tag = .r_bracket; - }, - '<' => if (result.loc.start != index) { - result.tag = .text; - } else { - continue :state .l_angle_bracket; - }, - '>' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - result.tag = .r_angle_bracket; - }, - '+' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - result.tag = .plus; - }, - '-' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; result.tag = .minus; }, - '\n' => if (result.loc.start != index) { - result.tag = .text; - } else { - index += 1; - result.tag = .newline; - }, - else => { - index += 1; - continue :state .default; - }, - }, - .invalid => { - index += 1; - switch (this.buffer[index]) { - 0 => if (index == this.buffer.len) { - result.tag = .invalid; - } else { - continue :state .invalid; - }, - else => continue :state .invalid, - } - }, - // referencing - .at_sign => { - index += 1; - switch (this.buffer[index]) { - 'a'...'z', 'A'...'Z', '0'...'9', '-', '.', ':', '/' => continue :state .at_sign, - ' ', '\n', '\t', '\r', ']', '|', 0 => result.tag = .reference, - else => continue :state .invalid, - } - }, - .l_angle_bracket => { - index += 1; - switch (this.buffer[index]) { - 'a'...'z', 'A'...'Z', '0'...'9', '-', '.', ':', '/' => continue :state .l_angle_bracket, - '>' => { - index += 1; - result.tag = .anchor; - }, - else => continue :state .invalid, - } - }, - // links - .l_bracket => { - index += 1; - switch (this.buffer[index]) { - ':' => { - index += 1; - result.tag = .l_bracket_colon; - }, - '-' => { - index += 1; - result.tag = .l_bracket_minus; - }, - '/' => { - index += 1; - result.tag = .l_bracket_slash; - }, - else => { - index -= 1; - result.tag = .l_bracket; - }, - } - }, - .pipe => { - index += 1; - switch (this.buffer[index]) { - '=' => { - index += 1; - result.tag = .pipe_equal; - }, - else => result.tag = .pipe, - } - }, - .slash => { - index += 1; - switch (this.buffer[index]) { - '/' => if (result.loc.start != index) { - result.tag = .text; - index -= 1; - } else continue :state .comment, - else => continue :state .default, - } - }, - .comment => { - index += 1; - switch (this.buffer[index]) { - 0, '\n' => result.tag = .comment, - else => continue :state .comment, - } - }, - .block => { - index += 1; - switch (this.buffer[index]) { - 0 => result.tag = .invalid, - '#' => if (this.buffer[index + 1] == '#') { - index += 2; - result.tag = .block; - } else { - continue :state .block; - }, - else => continue :state .block, - } - }, - else => { - print("Not yet implemented at {d}: '{s}'", .{ index, this.buffer[result.loc.start..index] }); - unreachable; - }, // not yet implemented - } + else => result.tag = .minus, + } + }, + // else => { + // print("Not yet implemented at {d}: '{s}'", .{ index, this.buffer[result.loc.start .. index + 1] }); + // unreachable; + // }, // not yet implemented } + result.loc.end = index; return result; } @@ -582,30 +318,12 @@ test "paragraphs" { try testTokenize( \\This is the first paragraph. \\Another line. - , &.{ .text, .newline, .text }); + , &.{.text}); try testTokenize( \\This is the first paragraph. \\This is still part of the paragraph. \\ - , &.{ .text, .newline, .text, .newline }); - try testTokenize( - \\This is the first paragraph. - \\This is still part of the paragraph. - \\ - \\This is the second paragraph. // comment - \\ - \\This is the third paragraph. - , &.{ .text, .newline, .text, .newline, .newline, .text, .comment, .newline, .newline, .text }); - try testTokenize( - \\This is the first paragraph. - \\This is still part of the paragraph. - \\ - \\// comment before a newline - \\This is the second paragraph. - \\// comment after a newline - \\ - \\This is the third paragraph. - , &.{ .text, .newline, .text, .newline, .newline, .comment, .newline, .text, .newline, .comment, .newline, .newline, .text }); + , &.{.text}); try testTokenize( \\This is the first paragraph. \\This is still part of the paragraph. @@ -616,380 +334,83 @@ test "paragraphs" { \\// comment after a newline \\ \\This is the third paragraph. - , &.{ .text, .newline, .text, .newline, .newline, .comment, .newline, .newline, .text, .newline, .comment, .newline, .newline, .text }); + , &.{.text}); try testTokenize( \\This is the first paragraph. \\This is still part of the paragraph. \\ - \\// comment before a newline - \\This is the second paragraph. \\--- \\This is the third paragraph. - \\// comment after a newline \\ - \\This is the last paragraph. - , &.{ .text, .newline, .text, .newline, .newline, .comment, .newline, .text, .newline, .minus, .minus, .minus, .newline, .text, .newline, .comment, .newline, .newline, .text }); + \\This is the last paragraph. + , &.{ .text, .separator, .text }); } test "styling" { - try testTokenize("*Test with _same_ more text*", &.{ .text, .underscore, .text, .underscore, .text }); - try testTokenize("*~_test_~*", &.{ .text, .underscore, .text, .underscore, .text }); - try testTokenize("*`~_test_~`*", &.{.text}); + try testTokenize("*Test with _same_ more text*", &.{ .asterik, .text, .underscore, .text, .underscore, .text, .asterik }); + try testTokenize("*`~_test_~`*", &.{ .asterik, .tick, .text, .underscore, .text, .underscore, .text, .tick, .asterik }); } -test "valid tracing" { - // target - try testTokenize("", &.{.anchor}); - try testTokenize("", &.{.anchor}); - try testTokenize("", &.{.anchor}); - try testTokenize("", &.{.anchor}); - try testTokenize("", &.{.anchor}); - // reference - try testTokenize("@anchor1", &.{.reference}); - try testTokenize("@a.n.c.h.o.r.", &.{.reference}); - try testTokenize("@a/n/c/h/o/r", &.{.reference}); - try testTokenize("@a:n:c:h:o:r", &.{.reference}); - try testTokenize("@a-n-c-h-o-r", &.{.reference}); +test "links" { // usage in link context - try testTokenize("##math x^2## ", &.{ .block, .text, .anchor }); - try testTokenize("@a|some text", &.{ .reference, .pipe, .text }); // see complete example below - try testTokenize("[:ul1p.n@builtin-functions|Builtin functions]", &.{ .l_bracket_colon, .text, .reference, .pipe, .text, .r_bracket }); + // try testTokenize("##math x^2## ", &.{ .block, .text, .anchor }); + // try testTokenize("@a|some text", &.{ .reference, .pipe, .text }); // see complete example below + // try testTokenize("[:ul1p.n@builtin-functions|Builtin functions]", &.{ .l_bracket_colon, .text, .reference, .pipe, .text, .r_bracket }); } -test "invalid anchors" { - // target - try testTokenize("", &.{.invalid}); - try testTokenize("", &.{.invalid}); - // reference - try testTokenize("@a_n_c_h_o_r", &.{.invalid}); +test "quotes" { + try testTokenize( + \\> Forty hour work weeks are a relic of the Industrial Age. Knowledge works function like athletes - train and sprint, then rest and reassess. + \\> -- Naval Ravikant + , &.{.quote}); + try testTokenize( + \\> Forty hour work weeks are a relic of the Industrial Age. Knowledge works function like athletes - train and sprint, then rest and reassess.-- Naval Ravikant + , &.{.quote}); } test "blocks" { - try testTokenize("##zig fn main() void {}##", &.{.block}); - try testTokenize("This is some text ##zig fn main() void {}## with inline code.", &.{ .text, .block, .text }); - try testTokenize("##zig fn main() void {}## with inline code.", &.{ .block, .text }); - try testTokenize( - \\##quote - \\Forty hour work weeks are a relic of the Industrial Age. Knowledge works function like athletes - train and sprint, then rest and reassess. - \\-- Naval Ravikant - \\## - , &.{.block}); - try testTokenize( - \\##quote - \\Forty hour work weeks are a relic of the Industrial Age. Knowledge works function like athletes - train and sprint, then rest and reassess. - \\-- Naval Ravikant - \\## With some text afterwards - , &.{ .block, .text }); - try testTokenize("##math x^2## Math", &.{ .block, .text }); - try testTokenize( - \\##math - \\sum(i)^(n)_(i = 0) = (n^2 + n) / 2 - \\## - , &.{.block}); -} - -test "comment" { - try testTokenize("// This is a comment", &.{.comment}); - try testTokenize( - \\// This is a comment - \\// Another comment (which is not treated the same as the one above) - , - &.{ .comment, .newline, .comment }, - ); - try testTokenize( - \\This is simple paragraph with - \\// comments in between - \\followed by some more normal paragraph // with a comment afterwards - \\// Ending with another comment in a new line - , - &.{ .text, .newline, .comment, .newline, .text, .comment, .newline, .comment }, - ); -} - -test "slash with paragraph" { - try testTokenize("/ with some paragraph", &.{.text}); - try testTokenize("they / them", &.{.text}); - try testTokenize( - \\/ with some paragraph - \\followed by some / more paragraph - , - &.{ .text, .newline, .text }, - ); + try testTokenize("```zig fn main() void {}```", &.{.block}); + try testTokenize("This is some text ```zig fn main() void {}``` with inline code.", &.{ .text, .block, .text }); + try testTokenize("```zig fn main() void {}``` with inline code.", &.{ .block, .text }); } test "heading" { - try testTokenize("= Heading 1", &.{ .equal, .text }); - try testTokenize("== Heading 2", &.{ .equal, .equal, .text }); - try testTokenize("=== Heading 3", &.{ .equal, .equal, .equal, .text }); - try testTokenize("==== Heading 4", &.{ .equal, .equal, .equal, .equal, .text }); - try testTokenize("===== Heading 5", &.{ .equal, .equal, .equal, .equal, .equal, .text }); - try testTokenize("====== Heading 6", &.{ .equal, .equal, .equal, .equal, .equal, .equal, .text }); - try testTokenize("=| test", &.{ .equal, .pipe, .text }); + try testTokenize("# Heading 1", &.{.heading}); + try testTokenize("## Heading 2", &.{.heading}); + try testTokenize("### Heading 3", &.{.heading}); + try testTokenize("#### Heading 4", &.{.heading}); + try testTokenize("##### Heading 5", &.{.heading}); + try testTokenize("###### Heading 6", &.{.heading}); try testTokenize( - \\= Heading With some paragraph // with a comment - \\ Followed with some more paragraph for that heading. - , &.{ .equal, .text, .comment, .newline, .text }); - try testTokenize( - \\= Heading With some paragraph - \\ Followed with some more paragraph for that heading. - , &.{ .equal, .text, .anchor, .newline, .text }); - try testTokenize( - \\= Heading With some paragraph // With an additional comment - \\ Followed with some more paragraph for that heading. - , &.{ .equal, .text, .anchor, .text, .comment, .newline, .text }); - try testTokenize( - \\= Heading With some paragraph // With an additional comment - \\ Followed with some more paragraph for that heading. - , &.{ .equal, .text, .comment, .newline, .text }); - try testTokenize("Some paragraph with some equal signs=Which is not a heading.", &.{ .text, .equal, .text }); - try testTokenize("Some paragraph with some equal signs =Which is not a heading.", &.{ .text, .equal, .text }); - try testTokenize("Some paragraph with some equal signs = Which is not a heading.", &.{ .text, .equal, .text }); + \\# Heading With some paragraph + \\ + \\Followed with some more paragraph for that heading. + , &.{ .heading, .text }); + try testTokenize("Some paragraph with some equal signs#Which is not a heading.", &.{ .text, .hashtag, .text }); + try testTokenize("Some paragraph with some equal signs #Which is not a heading.", &.{ .text, .hashtag, .text }); + try testTokenize("Some paragraph with some equal signs # Which is not a heading.", &.{ .text, .heading }); } test "lists" { - try testTokenize( - \\- First level - \\-- Second level - \\-- Second level - \\- First level - \\-- Second level - \\--- Third level - , &.{ .minus, .text, .newline, .minus, .minus, .text, .newline, .minus, .minus, .text, .newline, .minus, .text, .newline, .minus, .minus, .text, .newline, .minus, .minus, .minus, .text }); - try testTokenize( - \\+ First level - \\++ Second level - \\++ Second level - \\+ First level - \\++ Second level - \\+++ Third level - , &.{ .plus, .text, .newline, .plus, .plus, .text, .newline, .plus, .plus, .text, .newline, .plus, .text, .newline, .plus, .plus, .text, .newline, .plus, .plus, .plus, .text }); + // try testTokenize( + // \\- First level + // \\ - Second level + // \\ - Second level + // \\- First level + // \\ - Second level + // \\ - Third level + // , &.{ .minus, .text, .minus, .minus, .text, .minus, .minus, .text, .minus, .text, .minus, .minus, .text, .minus, .minus, .minus, .text }); } -test "tables" { +test "tables are just text" { try testTokenize( - \\|= Build Mode |= Runtime Safety |= Optimizations | + \\| Build Mode | Runtime Safety | Optimizations | + \\| | | | \\| Debug (default) | Yes | No | \\| ReleaseSafe | Yes | Yes, Speed | \\| ReleaseSmall | No | Yes, Size | \\| ReleaseFast | No | Yes, Speed | - , &.{ - .pipe_equal, .text, .pipe_equal, .text, .pipe_equal, .text, .pipe, .newline, - .pipe, .text, .pipe, .text, .pipe, .text, .pipe, .newline, - .pipe, .text, .pipe, .text, .pipe, .text, .pipe, .newline, - .pipe, .text, .pipe, .text, .pipe, .text, .pipe, .newline, - .pipe, .text, .pipe, .text, .pipe, .text, .pipe, - }); -} - -test "Example note with table" { - try testTokenize( - \\:code:zig: - \\ - \\= Build modes - \\Zig provides different kind of build modes for different purposes. - \\ - \\|= Build Mode |= Runtime Safety |= Optimizations | - \\| Debug (default) | Yes | No | - \\| ReleaseSafe | Yes | Yes, Speed | - \\| ReleaseSmall | No | Yes, Size | - \\| ReleaseFast | No | Yes, Speed | - \\ - \\ - \\With runtime safety checks enabled the compiler asserts code to enable the detection of illegal behaviour during runtime. If such a check fails a call to `@panic` ([:ul1p.n@builtin-functions|Builtin functions]) will be emitted. - \\ - \\##fn [/https://ziglang.org/documentation/master/#Build-Mode]## - , &.{ - .colon, - .text, - .colon, - .text, - .colon, - .newline, - .newline, - .equal, - .text, - .anchor, - .newline, - .text, - .newline, - .newline, - .pipe_equal, - .text, - .pipe_equal, - .text, - .pipe_equal, - .text, - .pipe, - .newline, - .pipe, - .text, - .pipe, - .text, - .pipe, - .text, - .pipe, - .newline, - .pipe, - .text, - .pipe, - .text, - .pipe, - .text, - .pipe, - .newline, - .pipe, - .text, - .pipe, - .text, - .pipe, - .text, - .pipe, - .newline, - .pipe, - .text, - .pipe, - .text, - .pipe, - .text, - .pipe, - .newline, - .anchor, - .newline, - .newline, - .text, - .l_bracket_colon, - .text, - .reference, - .pipe, - .text, - .r_bracket, - .text, - .newline, - .newline, - .block, // footnote - .text, // whitespace - .anchor, - }); -} - -test "Example note with code snippets" { - try testTokenize( - \\:code:zig: - \\ - \\= Conditional Code - \\Controlling not just the control flow of the code, but also which parts of the code base are actually compiled and used when shipping the application is very crucial and often done via condiationally enabling / disabling code. They are usually controlled via _feature toggles_ and can be implemented in zig via [:ly9j.n|comptime] (pre-processor statements in C/C++, etc.). - \\ - \\[:ly9j.n@comptime] even allows mixing build and runtime checks, see the following example: - \\ - \\##zig - \\ fn myFunction() void { - \\ if (hasFeature()) { - \\ // Feature-specific code - \\ } else { - \\ // Default code - \\ } - \\ } - \\ - \\ inline fn hasFeature() bool { - \\ return (comptime comptimeCheck()) and runtimeCheck(); - \\ } - \\## - \\ - \\Both the [:g0ic.n@inline] and [:ly9j.n@comptime] keywords are required, such that the `hasFeature` function call in `myFunction` will be [:msev.n|correctly] evaluated during build-time. - \\ - \\Most commonly such conditional code is used to provide _platform specific_ implementations: - \\ - \\##zig - \\ const builtin = @import("builtin"); - \\ - \\ fn myFunction() void { - \\ if (builtin.os.tag == .macos) { - \\ // This code will only be included if the target OS is macOS. - \\ return; - \\ } - \\ - \\ // This code will be included for all other operating systems. - \\ } - \\## - \\ - \\##fn [/https://mitchellh.com/writing/zig-comptime-conditional-disable|Conditionally Disabling Code with comptime in Zig - Mitchell Hashimoto]## - , &.{ - .colon, - .text, - .colon, - .text, - .colon, - .newline, - .newline, - .equal, - .text, - .anchor, - .newline, - .text, - .underscore, - .text, - .underscore, - .text, - .l_bracket_colon, - .text, - .pipe, - .text, - .r_bracket, - .text, - .minus, - .text, - .plus, - .plus, - .text, - .newline, - .newline, - .l_bracket_colon, - .text, - .reference, - .r_bracket, - .text, - .colon, - .newline, - .newline, - .block, - .newline, - .newline, - .text, - .l_bracket_colon, - .text, - .reference, - .r_bracket, - .text, - .l_bracket_colon, - .text, - .reference, - .r_bracket, - .text, - .l_bracket_colon, - .text, - .pipe, - .text, - .r_bracket, - .text, - .minus, - .text, - .newline, - .newline, - .text, - .underscore, - .text, - .underscore, - .text, - .colon, - .newline, - .newline, - .block, - .newline, - .newline, - .block, // footnote - .text, // whitespace - .anchor, - }); + , &.{.text}); } /// Test tokenizer's iterator outputs for the provided source. It should diff --git a/src/root.zig b/src/root.zig index 5da6f52..4632931 100644 --- a/src/root.zig +++ b/src/root.zig @@ -3,7 +3,7 @@ ///! corresponding errors are returned. For detailed error messages refer to ///! `errorMessage()` pub const Ast = @import("Ast.zig"); -pub const parser = @import("parser.zig"); +// pub const parser = @import("parser.zig"); pub const lexer = @import("lexer.zig"); test {