224 lines
8.9 KiB
Go
224 lines
8.9 KiB
Go
package tokenize
|
|
|
|
import (
|
|
"git.makaay.nl/mauricem/go-parsekit/read"
|
|
)
|
|
|
|
// API holds the internal state of a tokenizer run. A run uses tokenize.Handler
|
|
// functions to move the tokenizer forward through the input and to provide
|
|
// tokenizer output. The API is used by these tokenize.Handler functions to:
|
|
//
|
|
// • access and process runes / bytes from the input data
|
|
//
|
|
// • flush processed input data that are not required anymore (FlushInput)
|
|
//
|
|
// • fork the API for easy lookahead support (Fork, Merge, Reset, Dispose)
|
|
//
|
|
// • emit tokens and/or bytes to be used by a parser
|
|
//
|
|
// BASIC OPERATION:
|
|
//
|
|
// To retrieve the next rune from the API, call the NextRune() method.
|
|
//
|
|
// When the rune is to be accepted as input, call the method Accept(). The rune
|
|
// is then added to the result runes of the API and the read cursor is moved
|
|
// forward.
|
|
//
|
|
// By invoking NextRune() + Accept() multiple times, the result can be extended
|
|
// with as many runes as needed. Runes collected this way can later on be
|
|
// retrieved using the method Runes().
|
|
//
|
|
// It is mandatory to call Accept() after retrieving a rune, before calling
|
|
// NextRune() again. Failing to do so will result in a panic.
|
|
//
|
|
// Next to adding runes to the result, it is also possible to modify the
|
|
// stored runes or to add lexical Tokens to the result. For all things
|
|
// concerning results, take a look at the Result struct, which
|
|
// can be accessed though the method Result().
|
|
//
|
|
// FORKING OPERATION FOR EASY LOOKEAHEAD SUPPORT:
|
|
//
|
|
// Sometimes, we must be able to perform a lookahead, which might either
|
|
// succeed or fail. In case of a failing lookahead, the state of the
|
|
// API must be brought back to the original state, so we can try
|
|
// a different route.
|
|
//
|
|
// The way in which this is supported, is by forking an API struct by
|
|
// calling method Fork(). This will return a forked child API, with
|
|
// empty result data, but using the same read cursor position as the
|
|
// forked parent.
|
|
//
|
|
// After forking, the same interface as described for BASIC OPERATION can be
|
|
// used to fill the results. When the lookahead was successful, then
|
|
// Merge() can be called on the forked child to append the child's results
|
|
// to the parent's results, and to move the read cursor position to that
|
|
// of the child.
|
|
//
|
|
// When the lookahead was unsuccessful, then the forked child API can
|
|
// disposed by calling Dispose() on the forked child. This is not mandatory.
|
|
// Garbage collection will take care of this automatically.
|
|
// The parent API was never modified, so it can safely be used after disposal
|
|
// as if the lookahead never happened.
|
|
//
|
|
// Opinionized note:
|
|
// Many tokenizers/parsers take a different approach on lookaheads by using
|
|
// peeks and by moving the read cursor position back and forth, or by putting
|
|
// read input back on the input stream. That often leads to code that is
|
|
// efficient, however, in my opinion, not very intuitive to read. It can also
|
|
// be tedious to get the cursor position back at the correct position, which
|
|
// can lead to hard to track bugs. I much prefer this forking method, since
|
|
// no bookkeeping has to be implemented when implementing a parser.
|
|
type API struct {
|
|
stackFrames []stackFrame // the stack frames, containing stack level-specific data
|
|
stackLevel int // the current stack level
|
|
stackFrame *stackFrame // the current stack frame
|
|
Input Input // provides input-related functionality
|
|
Output Output // provides output-related functionality
|
|
}
|
|
|
|
type stackFrame struct {
|
|
offset int // the read offset (relative to the start of the reader buffer) for this stack frame
|
|
column int // the column at which the cursor is (0-indexed)
|
|
line int // the line at which the cursor is (0-indexed)
|
|
bytesStart int // the starting point in the API.bytes slice for runes produced by this stack level
|
|
bytesEnd int // the end point in the API.bytes slice for runes produced by this stack level
|
|
tokenStart int // the starting point in the API.tokens slice for tokens produced by this stack level
|
|
tokenEnd int // the end point in the API.tokens slice for tokens produced by this stack level
|
|
|
|
// TODO
|
|
err error // can be used by a Handler to report a specific issue with the input
|
|
}
|
|
|
|
const initialStackDepth = 64
|
|
const initialTokenStoreLength = 64
|
|
const initialByteStoreLength = 1024
|
|
|
|
// NewAPI initializes a new API struct, wrapped around the provided input.
|
|
// For an overview of allowed inputs, take a look at the documentation
|
|
// for parsekit.read.New().
|
|
func NewAPI(input interface{}) *API {
|
|
api := &API{
|
|
stackFrames: make([]stackFrame, initialStackDepth),
|
|
}
|
|
api.Input = Input{
|
|
api: api,
|
|
reader: read.New(input),
|
|
}
|
|
api.Output = Output{
|
|
api: api,
|
|
data: make([]byte, initialByteStoreLength),
|
|
tokens: make([]Token, initialTokenStoreLength),
|
|
}
|
|
api.stackFrame = &api.stackFrames[0]
|
|
|
|
return api
|
|
}
|
|
|
|
// Fork forks off a child of the API struct. It will reuse the same
|
|
// read buffer and cursor position, but for the rest this can be considered
|
|
// a fresh API.
|
|
//
|
|
// By forking an API, you can freely work with the forked child, without
|
|
// affecting the parent API. This is for example useful when you must perform
|
|
// some form of lookahead.
|
|
//
|
|
// When processing of the Handler was successful and you want to add the results
|
|
// to the parent API, you can call Merge() on the forked child.
|
|
// This will add the results to the results of the parent (runes, tokens).
|
|
// It also updates the read cursor position of the parent to that of the child.
|
|
//
|
|
// When the lookahead was unsuccessful, then the forked child API can
|
|
// disposed by calling Dispose() on the forked child. This is not mandatory.
|
|
// Garbage collection will take care of this automatically.
|
|
// The parent API was never modified, so it can safely be used after disposal
|
|
// as if the lookahead never happened.
|
|
func (tokenAPI *API) Fork() int {
|
|
newStackLevel := tokenAPI.stackLevel + 1
|
|
newStackSize := newStackLevel + 1
|
|
|
|
// Grow the stack frames capacity when needed.
|
|
if cap(tokenAPI.stackFrames) < newStackSize {
|
|
newFrames := make([]stackFrame, newStackSize*2)
|
|
copy(newFrames, tokenAPI.stackFrames)
|
|
tokenAPI.stackFrames = newFrames
|
|
}
|
|
|
|
tokenAPI.stackLevel++
|
|
|
|
// This can be written in a shorter way, but this turned out to
|
|
// be the best way performance-wise.
|
|
parent := tokenAPI.stackFrame
|
|
child := &tokenAPI.stackFrames[tokenAPI.stackLevel]
|
|
child.offset = parent.offset
|
|
child.column = parent.column
|
|
child.line = parent.line
|
|
child.bytesStart = parent.bytesEnd
|
|
child.bytesEnd = parent.bytesEnd
|
|
child.tokenStart = parent.tokenEnd
|
|
child.tokenEnd = parent.tokenEnd
|
|
tokenAPI.stackFrame = child
|
|
|
|
return tokenAPI.stackLevel
|
|
}
|
|
|
|
// Merge appends the results of a forked child API (runes, tokens) to the
|
|
// results of its parent. The read cursor of the parent is also updated
|
|
// to that of the forked child.
|
|
//
|
|
// After the merge operation, the child results are reset so it can immediately
|
|
// be reused for performing another match. This means that all Result data are
|
|
// cleared, but the read cursor position is kept at its current position.
|
|
// This allows a child to feed results in chunks to its parent.
|
|
//
|
|
// Once the child is no longer needed, it can be disposed of by using the
|
|
// method Dispose(), which will return the tokenizer to the parent.
|
|
func (tokenAPI *API) Merge(stackLevel int) {
|
|
if stackLevel == 0 {
|
|
callerPanic("Merge", "tokenize.API.{name}(): {name}() called at {caller} "+
|
|
"on the top-level API stack level 0")
|
|
}
|
|
if stackLevel != tokenAPI.stackLevel {
|
|
callerPanic("Merge", "tokenize.API.{name}(): {name}() called at {caller} "+
|
|
"on API stack level %d, but the current stack level is %d "+
|
|
"(forgot to Dispose() a forked child?)", stackLevel, tokenAPI.stackLevel)
|
|
}
|
|
|
|
parent := &tokenAPI.stackFrames[stackLevel-1]
|
|
|
|
// The end of the parent slice aligns with the start of the child slice.
|
|
// Because of this, to merge the parent slice can simply be expanded
|
|
// to include the child slice.
|
|
// parent : |----------|
|
|
// child: |------|
|
|
// After merge operation:
|
|
// parent: |-----------------|
|
|
// child: |---> continue reading from here
|
|
parent.bytesEnd = tokenAPI.stackFrame.bytesEnd
|
|
tokenAPI.stackFrame.bytesStart = tokenAPI.stackFrame.bytesEnd
|
|
|
|
// The same logic applies to tokens.
|
|
parent.tokenEnd = tokenAPI.stackFrame.tokenEnd
|
|
tokenAPI.stackFrame.tokenStart = tokenAPI.stackFrame.tokenEnd
|
|
|
|
parent.offset = tokenAPI.stackFrame.offset
|
|
parent.line = tokenAPI.stackFrame.line
|
|
parent.column = tokenAPI.stackFrame.column
|
|
|
|
tokenAPI.stackFrame.err = nil
|
|
}
|
|
|
|
func (tokenAPI *API) Dispose(stackLevel int) {
|
|
if stackLevel == 0 {
|
|
callerPanic("Dispose", "tokenize.API.{name}(): {name}() called at {caller} "+
|
|
"on the top-level API stack level 0")
|
|
}
|
|
if stackLevel != tokenAPI.stackLevel {
|
|
callerPanic("Dispose", "tokenize.API.{name}(): {name}() called at {caller} "+
|
|
"on API stack level %d, but the current stack level is %d "+
|
|
"(forgot to Dispose() a forked child?)", stackLevel, tokenAPI.stackLevel)
|
|
}
|
|
|
|
tokenAPI.stackLevel = stackLevel - 1
|
|
tokenAPI.stackFrame = &tokenAPI.stackFrames[stackLevel-1]
|
|
}
|