diff --git a/tokenize/api.go b/tokenize/api.go index 37b7512..8faa4ae 100644 --- a/tokenize/api.go +++ b/tokenize/api.go @@ -215,10 +215,14 @@ func (i *API) Fork() int { parent := i.stackFrame - i.stackFrame = &i.stackFrames[i.stackLevel] - *i.stackFrame = *parent - i.stackFrame.runeStart = parent.runeEnd - i.stackFrame.tokenStart = parent.tokenEnd + f := &i.stackFrames[i.stackLevel] + f.offset = parent.offset + f.cursor = parent.cursor + f.runeStart = parent.runeEnd + f.runeEnd = parent.runeEnd + f.tokenStart = parent.tokenEnd + f.tokenEnd = parent.tokenEnd + i.stackFrame = f return i.stackLevel } diff --git a/tokenize/cursor.go b/tokenize/cursor.go index db8af3b..1f7c02c 100644 --- a/tokenize/cursor.go +++ b/tokenize/cursor.go @@ -2,13 +2,10 @@ package tokenize import ( "fmt" - "unicode/utf8" ) // Cursor represents the position of a cursor in various ways. type Cursor struct { - Byte int // The cursor offset in bytes - Rune int // The cursor offset in UTF8 runes Column int // The column at which the cursor is (0-indexed) Line int // The line at which the cursor is (0-indexed) } @@ -33,8 +30,6 @@ func (c *Cursor) move(input string) *Cursor { } func (c *Cursor) moveByRune(r rune) *Cursor { - c.Byte += utf8.RuneLen(r) - c.Rune++ if r == '\n' { c.Column = 0 c.Line++ diff --git a/tokenize/cursor_test.go b/tokenize/cursor_test.go index 893dd61..5fba54c 100644 --- a/tokenize/cursor_test.go +++ b/tokenize/cursor_test.go @@ -53,12 +53,6 @@ func TestGivenCursor_WhenMoving_CursorIsUpdated(t *testing.T) { for _, s := range test.input { c.move(s) } - if c.Byte != test.byte { - t.Errorf("[%s] Unexpected byte offset %d (expected %d)", test.name, c.Byte, test.byte) - } - if c.Rune != test.rune { - t.Errorf("[%s] Unexpected rune offset %d (expected %d)", test.name, c.Rune, test.rune) - } if c.Line != test.line { t.Errorf("[%s] Unexpected line offset %d (expected %d)", test.name, c.Line, test.line) } diff --git a/tokenize/handlers_builtin_test.go b/tokenize/handlers_builtin_test.go index 9c90ab0..e436461 100644 --- a/tokenize/handlers_builtin_test.go +++ b/tokenize/handlers_builtin_test.go @@ -129,7 +129,7 @@ func TestAtoms(t *testing.T) { {"bb", a.RuneRange('b', 'e'), true, "b"}, {"cc", a.RuneRange('b', 'e'), true, "c"}, {"", a.EndOfFile, true, ""}, - {"⌘", a.AnyRune, true, "⌘"}, + {"😂", a.AnyRune, true, "😂"}, {"\xbc with AnyRune", a.AnyRune, true, "�"}, {"", a.AnyRune, false, ""}, {"⌘", a.ValidRune, true, "⌘"}, diff --git a/tokenize/tokenizer_whitebox_test.go b/tokenize/tokenizer_whitebox_test.go index 6f26b15..2ad3ad7 100644 --- a/tokenize/tokenizer_whitebox_test.go +++ b/tokenize/tokenizer_whitebox_test.go @@ -12,9 +12,7 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) { AssertEqual(t, "T", i.String(), "accepted rune in input") // Fork child := i.Fork() - AssertEqual(t, 1, i.stackFrame.cursor.Byte, "parent cursor.Byte") AssertEqual(t, 1, i.stackFrame.offset, "parent offset") - AssertEqual(t, 1, i.stackFrame.cursor.Byte, "child cursor.Byte") AssertEqual(t, 1, i.stackFrame.offset, "child offset") // Accept two runes via fork. i.NextRune() @@ -22,15 +20,12 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) { i.NextRune() i.Accept() // s AssertEqual(t, "es", i.String(), "result runes in fork") - AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].cursor.Byte, "parent cursor.Byte") AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset") - AssertEqual(t, 3, i.stackFrame.cursor.Byte, "child cursor.Byte") AssertEqual(t, 3, i.stackFrame.offset, "child offset") // Merge fork back into parent i.Merge(child) i.Dispose(child) AssertEqual(t, "Tes", i.String(), "result runes in parent Input after Merge()") - AssertEqual(t, 3, i.stackFrame.cursor.Byte, "parent cursor.Byte") AssertEqual(t, 3, i.stackFrame.offset, "parent offset") }