Speeding up the code some more. Big step was made by simplifying the cursor, continuing with that in the next commit.

tokenize/api.go

@@ -215,10 +215,14 @@ func (i *API) Fork() int {
 
 	parent := i.stackFrame
 
-	i.stackFrame = &i.stackFrames[i.stackLevel]
-	*i.stackFrame = *parent
-	i.stackFrame.runeStart = parent.runeEnd
-	i.stackFrame.tokenStart = parent.tokenEnd
+	f := &i.stackFrames[i.stackLevel]
+	f.offset = parent.offset
+	f.cursor = parent.cursor
+	f.runeStart = parent.runeEnd
+	f.runeEnd = parent.runeEnd
+	f.tokenStart = parent.tokenEnd
+	f.tokenEnd = parent.tokenEnd
+	i.stackFrame = f
 
 	return i.stackLevel
 }

tokenize/cursor.go

@@ -2,13 +2,10 @@ package tokenize
 
 import (
 	"fmt"
-	"unicode/utf8"
 )
 
 // Cursor represents the position of a cursor in various ways.
 type Cursor struct {
-	Byte   int // The cursor offset in bytes
-	Rune   int // The cursor offset in UTF8 runes
 	Column int // The column at which the cursor is (0-indexed)
 	Line   int // The line at which the cursor is (0-indexed)
 }
@@ -33,8 +30,6 @@ func (c *Cursor) move(input string) *Cursor {
 }
 
 func (c *Cursor) moveByRune(r rune) *Cursor {
-	c.Byte += utf8.RuneLen(r)
-	c.Rune++
 	if r == '\n' {
 		c.Column = 0
 		c.Line++

tokenize/cursor_test.go

@@ -53,12 +53,6 @@ func TestGivenCursor_WhenMoving_CursorIsUpdated(t *testing.T) {
 		for _, s := range test.input {
 			c.move(s)
 		}
-		if c.Byte != test.byte {
-			t.Errorf("[%s] Unexpected byte offset %d (expected %d)", test.name, c.Byte, test.byte)
-		}
-		if c.Rune != test.rune {
-			t.Errorf("[%s] Unexpected rune offset %d (expected %d)", test.name, c.Rune, test.rune)
-		}
 		if c.Line != test.line {
 			t.Errorf("[%s] Unexpected line offset %d (expected %d)", test.name, c.Line, test.line)
 		}

tokenize/handlers_builtin_test.go

@@ -129,7 +129,7 @@ func TestAtoms(t *testing.T) {
 		{"bb", a.RuneRange('b', 'e'), true, "b"},
 		{"cc", a.RuneRange('b', 'e'), true, "c"},
 		{"", a.EndOfFile, true, ""},
-		{"⌘", a.AnyRune, true, "⌘"},
+		{"😂", a.AnyRune, true, "😂"},
 		{"\xbc with AnyRune", a.AnyRune, true, "<22>"},
 		{"", a.AnyRune, false, ""},
 		{"⌘", a.ValidRune, true, "⌘"},

tokenize/tokenizer_whitebox_test.go

@@ -12,9 +12,7 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
 	AssertEqual(t, "T", i.String(), "accepted rune in input")
 	// Fork
 	child := i.Fork()
-	AssertEqual(t, 1, i.stackFrame.cursor.Byte, "parent cursor.Byte")
 	AssertEqual(t, 1, i.stackFrame.offset, "parent offset")
-	AssertEqual(t, 1, i.stackFrame.cursor.Byte, "child cursor.Byte")
 	AssertEqual(t, 1, i.stackFrame.offset, "child offset")
 	// Accept two runes via fork.
 	i.NextRune()
@@ -22,15 +20,12 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
 	i.NextRune()
 	i.Accept() // s
 	AssertEqual(t, "es", i.String(), "result runes in fork")
-	AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].cursor.Byte, "parent cursor.Byte")
 	AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset")
-	AssertEqual(t, 3, i.stackFrame.cursor.Byte, "child cursor.Byte")
 	AssertEqual(t, 3, i.stackFrame.offset, "child offset")
 	// Merge fork back into parent
 	i.Merge(child)
 	i.Dispose(child)
 	AssertEqual(t, "Tes", i.String(), "result runes in parent Input after Merge()")
-	AssertEqual(t, 3, i.stackFrame.cursor.Byte, "parent cursor.Byte")
 	AssertEqual(t, 3, i.stackFrame.offset, "parent offset")
 }