93 lines
2.7 KiB
Go
93 lines
2.7 KiB
Go
package tokenize
|
|
|
|
// Input provides input-related functionality for the tokenize API.
|
|
type ByteMode struct {
|
|
api *API
|
|
}
|
|
|
|
// Peek returns the byte at the provided byte offset.
|
|
//
|
|
// When an error occurs during reading the input, an error will be returned.
|
|
// When an offset is requested that is beyond the length of the available input
|
|
// data, then the error will be io.EOF.
|
|
func (byteMode ByteMode) Peek(offset int) (byte, error) {
|
|
a := byteMode.api
|
|
return a.reader.ByteAt(a.stackFrame.offset + offset)
|
|
}
|
|
|
|
// Skip is used to skip over one or more bytes that were read from the input.
|
|
// This tells the tokenizer: "I've seen these bytes. They are of no interest.
|
|
// I will now continue reading after these bytes."
|
|
//
|
|
// This will merely update the position of the cursor (which keeps track of what
|
|
// line and column we are on in the input data). The bytes are not added to
|
|
// the output.
|
|
//
|
|
// After the call, byte offset 0 for PeekByte() and PeekRune() will point at
|
|
// the first byte after the skipped bytes.
|
|
func (byteMode ByteMode) Skip(b byte) {
|
|
f := byteMode.api.stackFrame
|
|
f.moveCursorByByte(b)
|
|
f.offset++
|
|
}
|
|
|
|
func (byteMode ByteMode) SkipMulti(bytes ...byte) {
|
|
f := byteMode.api.stackFrame
|
|
for _, b := range bytes {
|
|
f.moveCursorByByte(b)
|
|
f.offset++
|
|
}
|
|
}
|
|
|
|
func (byteMode ByteMode) Accept(b byte) {
|
|
byteMode.Append(b)
|
|
byteMode.Skip(b)
|
|
}
|
|
|
|
func (byteMode ByteMode) Append(b byte) {
|
|
a := byteMode.api
|
|
f := a.stackFrame
|
|
a.growOutputData(f.bytesEnd + 1)
|
|
a.outputData[f.bytesEnd] = b
|
|
f.bytesEnd++
|
|
}
|
|
|
|
// AcceptMulti is used to accept one or more bytes that were read from the input.
|
|
// This tells the tokenizer: "I've seen these bytes. I want to make use of them
|
|
// for the final output, so please remember them for me. I will now continue
|
|
// reading after these bytes."
|
|
//
|
|
// This will update the position of the cursor (which keeps track of what line
|
|
// and column we are on in the input data) and add the bytes to the tokenizer
|
|
// output.
|
|
//
|
|
// After the call, byte offset 0 for PeekByte() and PeekRune() will point at
|
|
// the first byte after the accepted bytes.
|
|
func (byteMode ByteMode) AcceptMulti(bytes ...byte) {
|
|
byteMode.AppendMulti(bytes...)
|
|
byteMode.SkipMulti(bytes...)
|
|
}
|
|
|
|
func (byteMode ByteMode) AppendMulti(bytes ...byte) {
|
|
a := byteMode.api
|
|
f := a.stackFrame
|
|
curBytesEnd := f.bytesEnd
|
|
newBytesEnd := curBytesEnd + len(bytes)
|
|
|
|
a.growOutputData(newBytesEnd)
|
|
copy(a.outputData[curBytesEnd:], bytes)
|
|
f.bytesEnd = newBytesEnd
|
|
}
|
|
|
|
func (api *API) dataAddByte(b byte) {
|
|
curBytesEnd := api.stackFrame.bytesEnd
|
|
api.growOutputData(curBytesEnd + 1)
|
|
api.outputData[curBytesEnd] = b
|
|
api.stackFrame.bytesEnd++
|
|
}
|
|
|
|
func (api *API) dataSetBytes(bytes ...byte) {
|
|
api.dataClear()
|
|
api.Byte.AppendMulti(bytes...)
|
|
}
|