From ddd0ed49f6ca1f923894542df225ce666098e69e Mon Sep 17 00:00:00 2001
From: Maurice Makaay <maurice@makaay.nl>
Date: Tue, 16 Jul 2019 12:19:50 +0000
Subject: [PATCH] Don't resize the stack slices, since we keep track of their
 starts and ends anyway.

---
 tokenize/api.go | 43 ++++++++++++-------------------------------
 1 file changed, 12 insertions(+), 31 deletions(-)

diff --git a/tokenize/api.go b/tokenize/api.go
index e68b7c9..eb9948f 100644
--- a/tokenize/api.go
+++ b/tokenize/api.go
@@ -75,8 +75,8 @@ type API struct {
 	lastRuneWidth int          // the width in bytes of the last read rune
 	lastRuneErr   error        // the error for the last NextRune() call
 	runeRead      bool         // whether or not a rune was read using NextRune()
-	runes         []rune       // the rune stack
-	tokens        []Token      // the token stack
+	runes         []rune       // accepted runes
+	tokens        []Token      // accepted tokens
 	stackFrames   []stackFrame // the stack frames, containing stack level-specific data
 	stackLevel    int          // the current stack level
 	stackFrame    *stackFrame  // the current stack frame
@@ -105,9 +105,9 @@ const initialRuneStoreLength = 128
 func NewAPI(input interface{}) *API {
 	api := &API{
 		reader:      read.New(input),
-		runes:       make([]rune, 0, initialRuneStoreLength),
-		tokens:      make([]Token, 0, initialTokenStoreLength),
-		stackFrames: make([]stackFrame, 1, initialStackDepth),
+		runes:       make([]rune, initialRuneStoreLength),
+		tokens:      make([]Token, initialTokenStoreLength),
+		stackFrames: make([]stackFrame, initialStackDepth),
 	}
 	api.stackFrame = &api.stackFrames[0]
 
@@ -176,11 +176,9 @@ func (i *API) acceptBytes(bytes ...byte) {
 
 	// Grow the runes capacity when needed.
 	if cap(i.runes) < newRuneEnd {
-		newRunes := make([]rune, newRuneEnd, newRuneEnd*2)
+		newRunes := make([]rune, newRuneEnd*2)
 		copy(newRunes, i.runes)
 		i.runes = newRunes
-	} else {
-		i.runes = i.runes[0:newRuneEnd]
 	}
 
 	for offset, b := range bytes {
@@ -198,11 +196,9 @@ func (i *API) acceptRunes(width int, runes ...rune) {
 
 	// Grow the runes capacity when needed.
 	if cap(i.runes) < newRuneEnd {
-		newRunes := make([]rune, newRuneEnd, newRuneEnd*2)
+		newRunes := make([]rune, newRuneEnd*2)
 		copy(newRunes, i.runes)
 		i.runes = newRunes
-	} else {
-		i.runes = i.runes[0:newRuneEnd]
 	}
 
 	for offset, r := range runes {
@@ -237,11 +233,9 @@ func (i *API) Fork() int {
 
 	// Grow the stack frames capacity when needed.
 	if cap(i.stackFrames) < newStackSize {
-		newFrames := make([]stackFrame, newStackSize, newStackSize*2)
+		newFrames := make([]stackFrame, newStackSize*2)
 		copy(newFrames, i.stackFrames)
 		i.stackFrames = newFrames
-	} else {
-		i.stackFrames = i.stackFrames[0:newStackSize]
 	}
 
 	i.stackLevel++
@@ -323,10 +317,7 @@ func (i *API) Dispose(stackLevel int) {
 
 	i.runeRead = false
 	i.stackLevel = stackLevel - 1
-	i.stackFrames = i.stackFrames[:stackLevel]
 	i.stackFrame = &i.stackFrames[stackLevel-1]
-	i.runes = i.runes[0:i.stackFrame.runeEnd]
-	i.tokens = i.tokens[0:i.stackFrame.tokenEnd]
 }
 
 func (i *API) Reset() {
@@ -377,7 +368,6 @@ func (i *API) Rune(offset int) rune {
 }
 
 func (i *API) ClearRunes() {
-	i.runes = i.runes[:i.stackFrame.runeStart]
 	i.stackFrame.runeEnd = i.stackFrame.runeStart
 }
 
@@ -385,11 +375,9 @@ func (i *API) SetRunes(runes ...rune) {
 	// Grow the runes capacity when needed.
 	newRuneEnd := i.stackFrame.runeStart + len(runes)
 	if cap(i.runes) < newRuneEnd {
-		newRunes := make([]rune, newRuneEnd, newRuneEnd*2)
+		newRunes := make([]rune, newRuneEnd*2)
 		copy(newRunes, i.runes)
 		i.runes = newRunes
-	} else {
-		i.runes = i.runes[0:newRuneEnd]
 	}
 
 	for offset, r := range runes {
@@ -402,11 +390,9 @@ func (i *API) AddRunes(runes ...rune) {
 	// Grow the runes capacity when needed.
 	newRuneEnd := i.stackFrame.runeEnd + len(runes)
 	if cap(i.runes) < newRuneEnd {
-		newRunes := make([]rune, newRuneEnd, newRuneEnd*2)
+		newRunes := make([]rune, newRuneEnd*2)
 		copy(newRunes, i.runes)
 		i.runes = newRunes
-	} else {
-		i.runes = i.runes[0:newRuneEnd]
 	}
 
 	for offset, r := range runes {
@@ -443,7 +429,6 @@ func (i *API) TokenValue(offset int) interface{} {
 }
 
 func (i *API) ClearTokens() {
-	i.tokens = i.tokens[:i.stackFrame.tokenStart]
 	i.stackFrame.tokenEnd = i.stackFrame.tokenStart
 }
 
@@ -451,11 +436,9 @@ func (i *API) SetTokens(tokens ...Token) {
 	// Grow the tokens capacity when needed.
 	newTokenEnd := i.stackFrame.tokenStart + len(tokens)
 	if cap(i.tokens) < newTokenEnd {
-		newTokens := make([]Token, newTokenEnd, newTokenEnd*2)
+		newTokens := make([]Token, newTokenEnd*2)
 		copy(newTokens, tokens)
 		i.tokens = newTokens
-	} else {
-		i.tokens = i.tokens[0:newTokenEnd]
 	}
 
 	for offset, t := range tokens {
@@ -468,11 +451,9 @@ func (i *API) AddTokens(tokens ...Token) {
 	// Grow the tokens capacity when needed.
 	newTokenEnd := i.stackFrame.tokenEnd + len(tokens)
 	if cap(i.tokens) < newTokenEnd {
-		newTokens := make([]Token, newTokenEnd, newTokenEnd*2)
+		newTokens := make([]Token, newTokenEnd*2)
 		copy(newTokens, i.tokens)
 		i.tokens = newTokens
-	} else {
-		i.tokens = i.tokens[0:newTokenEnd]
 	}
 
 	for offset, t := range tokens {