Skip to content

Commit

Permalink
Remove newline attribute from tokens and move the decision on where t…
Browse files Browse the repository at this point in the history 
…o split lines to layout::Window.

So that layout::Window can tell where summaries begin and end, two new meta-tokens were added to
delineate summaries: SummaryPreamble and SummaryEpilogue. The SummaryNewline TokenizerState was
renamed to SummaryEpilogue, and a new SummaryPreamble TokenizerState was added.
  • Loading branch information
@misson20000
misson20000 committed Nov 12, 2023
1 parent 9c3f423 commit f81bf4d
Show file tree
Hide file tree
Showing 21 changed files with 1,111 additions and 496 deletions.
254 changes: 149 additions & 105 deletions doc/tokenizer.graphml
View file

Large diffs are not rendered by default.

Binary file modified doc/tokenizer.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
8 changes: 5 additions & 3 deletions src/bin/layoutplayground.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ struct Line {
tokens: vec::Vec<token::Token>
}

impl layout::Line for Line {
impl layout::LineView for Line {
type TokenIterator = vec::IntoIter<token::Token>;
type BorrowingTokenIterator<'a> = std::slice::Iter<'a, token::Token>;

fn from_tokens(tokens: vec::Vec<token::Token>) -> Self {
fn from_line(line: layout::Line) -> Self {
let tokens: vec::Vec<token::Token> = line.to_tokens().collect();

Line {
indent: tokens[0].depth,
tokens,
Expand All @@ -41,7 +43,7 @@ fn main() {

window.resize(150);

for line in window.lines {
for line in window.line_views {
for _ in 0..line.indent {
print!(" ");
}
Expand Down
280 changes: 158 additions & 122 deletions src/logic/tokenizer.rs
View file

Large diffs are not rendered by default.

16 changes: 8 additions & 8 deletions src/logic/tokenizer_tests/content_display.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,20 @@
<!-- root's content doesn't get displayed -->
</node>
<tokens>
<null node="root" nl="true" />
<title node="root" nl="true" />
<null node="root" />
<title node="root" />
<indent>
<title node="hexdump" nl="false" />
<title node="hexdump" />
<indent>
<hexdump node="hexdump" begin="0x0" end="0x8" nl="true" />
<hexdump node="hexdump" begin="0x8" end="0x10" nl="true" />
<hexdump node="hexdump" extent="00:08" line="00:+8" />
<hexdump node="hexdump" extent="08:10" line="08:+8" />
</indent>

<title node="hexstring" nl="false" />
<title node="hexstring" />
<indent>
<hexstring node="hexstring" begin="0x0" end="0x10" nl="true" />
<hexstring node="hexstring" extent="00:10" />
</indent>
<null node="root" nl="true" cursor="true" />
<null node="root" cursor="true" />
</indent>
</tokens>
</testcase>
30 changes: 15 additions & 15 deletions src/logic/tokenizer_tests/formatting.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,26 @@
</node>
</node>
<tokens>
<null node="root" nl="true" />
<title node="root" nl="true" />
<null node="root" />
<title node="root" />
<indent>
<hexdump node="root" begin="0x0" end="0x10" nl="true" />
<title node="inline32" nl="false" />
<hexdump node="root" extent="00:10" line="00:+0x10" />
<title node="inline32" />
<indent>
<hexstring node="inline32" begin="0x0" end="0x4" nl="true" />
<hexstring node="inline32" extent="00:04" />
</indent>
<hexdump node="root" begin="0x14" end="0x20" nl="true" />
<title node="inlinedump" nl="false" />
<hexdump node="root" extent="14:20" line="10:+0x10" />
<title node="inlinedump" />
<indent>
<hexdump node="inlinedump" begin="0x0" end="0x8" nl="true" />
<hexdump node="inlinedump" begin="0x8" end="0x10" nl="true" />
<hexdump node="inlinedump" begin="0x10" end="0x18" nl="true" />
<hexdump node="inlinedump" begin="0x18" end="0x20" nl="true" />
<hexdump node="inlinedump" extent="00:08" line="00:+8" />
<hexdump node="inlinedump" extent="08:10" line="08:+8" />
<hexdump node="inlinedump" extent="10:18" line="10:+8" />
<hexdump node="inlinedump" extent="18:20" line="18:+8" />
</indent>
<hexdump node="root" begin="0x40" end="0x50" nl="true" />
<hexdump node="root" begin="0x50" end="0x60" nl="true" />
<hexdump node="root" begin="0x60" end="0x70" nl="true" />
<null node="root" nl="true" cursor="true" />
<hexdump node="root" extent="40:50" line="40:+0x10" />
<hexdump node="root" extent="50:60" line="50:+0x10" />
<hexdump node="root" extent="60:70" line="60:+0x10" />
<null node="root" cursor="true" />
</indent>
</tokens>
</testcase>
80 changes: 40 additions & 40 deletions src/logic/tokenizer_tests/nesting.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,69 +18,69 @@
</node>
</node>
<tokens>
<null node="root" nl="true" />
<title node="root" nl="true" />
<null node="root" />
<title node="root" />
<indent>
<null node="zs1" nl="true" />
<title node="zs1" nl="true" />
<null node="zs1" />
<title node="zs1" />
<indent>
<null node="zs1" nl="true" cursor="true" />
<null node="zs1" cursor="true" />
</indent>
<null node="zs2" nl="true" />
<title node="zs2" nl="true" />
<null node="zs2" />
<title node="zs2" />
<indent>
<null node="zs3" nl="true" />
<title node="zs3" nl="true" />
<null node="zs3" />
<title node="zs3" />
<indent>
<null node="zs3" nl="true" cursor="true" />
<null node="zs3" cursor="true" />
</indent>
<null node="zs2" nl="true" cursor="true" />
<null node="zs2" cursor="true" />
</indent>
<null node="zs4" nl="true" />
<title node="zs4" nl="true" />
<null node="zs4" />
<title node="zs4" />
<indent>
<null node="zs4" nl="true" cursor="true" />
<null node="zs4" cursor="true" />
</indent>
<null node="bumper" nl="true" />
<title node="bumper" nl="true" />
<null node="bumper" />
<title node="bumper" />
<indent>
<hexdump node="bumper" begin="0x0" end="0x10" nl="true" />
<null node="bumper" nl="true" cursor="true" />
<hexdump node="bumper" extent="00:10" line="00:+0x10" />
<null node="bumper" cursor="true" />
</indent>
<null node="nest1" nl="true" />
<title node="nest1" nl="true" />
<null node="nest1" />
<title node="nest1" />
<indent>
<null node="nest1.1" nl="true" />
<title node="nest1.1" nl="true" />
<null node="nest1.1" />
<title node="nest1.1" />
<indent>
<null node="nest1.1.1" nl="true" />
<title node="nest1.1.1" nl="true" />
<null node="nest1.1.1" />
<title node="nest1.1.1" />
<indent>
<hexdump node="nest1.1.1" begin="0x0" end="0x8" nl="true" />
<null node="nest1.1.1" nl="true" cursor="true" />
<hexdump node="nest1.1.1" extent="00:08" line="00:+0x10" />
<null node="nest1.1.1" cursor="true" />
</indent>
<null node="nest1.1" nl="true" cursor="true" />
<null node="nest1.1" cursor="true" />
</indent>
<null node="nest1.2" nl="true" />
<title node="nest1.2" nl="true" />
<null node="nest1.2" />
<title node="nest1.2" />
<indent>
<null node="nest1.2" nl="true" cursor="true" />
<null node="nest1.2" cursor="true" />
</indent>
<null node="nest1.3" nl="true" />
<title node="nest1.3" nl="true" />
<null node="nest1.3" />
<title node="nest1.3" />
<indent>
<null node="nest1.3.1" nl="true" />
<title node="nest1.3.1" nl="true" />
<null node="nest1.3.1" />
<title node="nest1.3.1" />
<indent>
<hexdump node="nest1.3.1" begin="0x0" end="0x8" nl="true" />
<null node="nest1.3.1" nl="true" cursor="true" />
<hexdump node="nest1.3.1" extent="00:08" line="00:+0x10" />
<null node="nest1.3.1" cursor="true" />
</indent>
<null node="nest1.3" nl="true" cursor="true" />
<null node="nest1.3" cursor="true" />
</indent>
<hexdump node="nest1" begin="0x10" end="0x18" nl="true" />
<null node="nest1" nl="true" cursor="true" />
<hexdump node="nest1" extent="10:18" line="10:+0x10" />
<null node="nest1" cursor="true" />
</indent>
<null node="root" nl="true" cursor="true" />
<null node="root" cursor="true" />
</indent>
</tokens>
</testcase>
30 changes: 15 additions & 15 deletions src/logic/tokenizer_tests/simple.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,24 @@
</node>
</node>
<tokens>
<null node="root" nl="true" />
<title node="root" nl="true" />
<null node="root" />
<title node="root" />
<indent>
<hexdump node="root" begin="0x0" end="0x10" nl="true" />
<hexdump node="root" begin="0x10" end="0x20" nl="true" />
<hexdump node="root" begin="0x20" end="0x30" nl="true" />
<hexdump node="root" begin="0x30" end="0x32" nl="true" />
<null node="child" nl="true" />
<title node="child" nl="true" />
<hexdump node="root" extent="00:10" line="00:+0x10" />
<hexdump node="root" extent="10:20" line="10:+0x10" />
<hexdump node="root" extent="20:30" line="20:+0x10" />
<hexdump node="root" extent="30:32" line="30:+0x10" />
<null node="child" />
<title node="child" />
<indent>
<hexdump node="child" begin="0x0" end="0x10" nl="true" />
<hexdump node="child" begin="0x10" end="0x18" nl="true" />
<null node="child" nl="true" cursor="true" />
<hexdump node="child" extent="00:10" line="00:+0x10" />
<hexdump node="child" extent="10:18" line="10:+0x10" />
<null node="child" cursor="true" />
</indent>
<hexdump node="root" begin="0x4a" end="0x50" nl="true" />
<hexdump node="root" begin="0x50" end="0x60" nl="true" />
<hexdump node="root" begin="0x60" end="0x70" nl="true" />
<null node="root" nl="true" cursor="true" />
<hexdump node="root" extent="4a:50" line="40:+0x10" />
<hexdump node="root" extent="50:60" line="50:+0x10" />
<hexdump node="root" extent="60:70" line="60:+0x10" />
<null node="root" cursor="true" />
</indent>
</tokens>
</testcase>
60 changes: 30 additions & 30 deletions src/logic/tokenizer_tests/summary.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,67 +13,67 @@
</node>
</node>
<tokens>
<null node="root" nl="true" />
<title node="root" nl="true" />
<null node="root" />
<title node="root" />
<indent>
<title node="entity" nl="false" />
<open node="entity" nl="false" cursor="true" />
<title node="entity" />
<preamble node="entity" />
<open node="entity" />

<summlabel node="type" nl="false" />
<summlabel node="type" />
<indent>
<hexdump node="type" begin="0x0" end="0x8" nl="false" />
<hexstring node="type" extent="00:08" />
</indent>

<comma node="entity" nl="false" />
<comma node="entity" />

<summlabel node="pos" nl="false" />
<summlabel node="pos" />
<indent>
<open node="pos" nl="false" cursor="true" />
<open node="pos" />

<summlabel node="x1" nl="false" />
<summlabel node="x1" />
<indent>
<hexstring node="x1" begin="0x0" end="0x8" nl="false" />
<hexstring node="x1" extent="00:08" />
</indent>

<comma node="pos" nl="false" />
<comma node="pos" />

<summlabel node="y1" nl="false" />
<summlabel node="y1" />
<indent>
<hexstring node="y1" begin="0x0" end="0x8" nl="false" />
<hexstring node="y1" extent="00:08" />
</indent>

<close node="pos" nl="false" cursor="true" />
<close node="pos" />
</indent>

<comma node="entity" nl="false" />
<comma node="entity" />

<summlabel node="target" nl="false" />
<summlabel node="target" />
<indent>
<open node="target" nl="false" cursor="true" />
<open node="target" />

<summlabel node="x2" nl="false" />
<summlabel node="x2" />
<indent>
<hexstring node="x2" begin="0x0" end="0x8" nl="false" />
<hexstring node="x2" extent="00:08" />
</indent>

<comma node="target" nl="false" />
<comma node="target" />

<summlabel node="y2" nl="false" />
<summlabel node="y2" />
<indent>
<hexstring node="y2" begin="0x0" end="0x8" nl="false" />
<hexstring node="y2" extent="00:08" />
</indent>

<close node="target" nl="false" cursor="true" />
<close node="target" />
</indent>

<close node="entity" nl="false" cursor="true" />

<null node="entity" nl="true" />
<close node="entity" />
<epilogue node="entity" />

<hexdump node="root" begin="0x30" end="0x40" nl="true" />
<hexdump node="root" begin="0x40" end="0x50" nl="true" />
<hexdump node="root" extent="30:40" line="30:+0x10" />
<hexdump node="root" extent="40:50" line="40:+0x10" />

<null node="root" nl="true" cursor="true" />
<null node="root" cursor="true" />
</indent>
</tokens>
</testcase>
2 changes: 1 addition & 1 deletion src/model/addr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ impl Extent {
other.begin >= self.begin && other.end <= self.end
}

/// Returns the intersection of this extent and the other, if they overlap. If they abut, None is returned.
/// Returns the intersection of this extent and the other, if they overlap. If they abut or don't overlap, None is returned.
pub fn intersection(&self, other: Extent) -> Option<Extent> {
let begin = std::cmp::max(self.begin, other.begin);
let end = std::cmp::min(self.end, other.end);
Expand Down
Loading

0 comments on commit f81bf4d

Please sign in to comment.