103 lines
3.5 KiB
Go
103 lines
3.5 KiB
Go
package tokenize
|
|
|
|
// InputByteMode provides byte-driven input/output functionality for the tokenize API.
|
|
type InputByteMode struct {
|
|
api *API
|
|
}
|
|
|
|
// Peek returns the byte at the provided byte offset.
|
|
//
|
|
// When an error occurs during reading the input, an error will be returned.
|
|
// When an offset is requested that is beyond the length of the available input
|
|
// data, then the error will be io.EOF.
|
|
func (byteMode InputByteMode) Peek(offset int) (byte, error) {
|
|
a := byteMode.api
|
|
return a.reader.ByteAt(a.pointers.offset + offset)
|
|
}
|
|
|
|
// PeekMulti returns at max the provided maximum number of bytes at the provided
|
|
// byte offset. When less bytes are available on the input, then this is not an
|
|
// error as such. The returned error can in such case be set to io.EOF to indicate
|
|
// that the end of the input was reached though.
|
|
func (byteMode InputByteMode) PeekMulti(offset int, count int) ([]byte, error) {
|
|
a := byteMode.api
|
|
return a.reader.BytesAt(a.pointers.offset+offset, count)
|
|
}
|
|
|
|
// PeekBuffered returns the full buffered input from the reader, starting at the
|
|
// provided byte offset. When the offset is currently not available in the buffer,
|
|
// the reader buffer will be filled to make it available.
|
|
//
|
|
// When less bytes are available on the input than the requested byte offset,
|
|
// the returned error will be io.EOF to indicate that the end of the input was
|
|
// reached.
|
|
func (byteMode InputByteMode) PeekBuffered(offset int) ([]byte, error) {
|
|
a := byteMode.api
|
|
return a.reader.BufferedBytesAt(a.pointers.offset + offset)
|
|
}
|
|
|
|
func (byteMode InputByteMode) Accept(b byte) {
|
|
byteMode.AddByteToOutput(b)
|
|
byteMode.MoveCursor(b)
|
|
}
|
|
|
|
func (byteMode InputByteMode) AddByteToOutput(b byte) {
|
|
a := byteMode.api
|
|
if a.Output.suspended == 0 {
|
|
byteMode.api.Output.AddByte(b)
|
|
}
|
|
}
|
|
|
|
// AcceptMulti accepts one or more bytes that were read from the input.
|
|
// This tells the tokenizer: "I've seen these bytes. I want to make use of them
|
|
// for the final output, so please remember them for me. I will now continue
|
|
// reading after these bytes."
|
|
//
|
|
// This will update the position of the cursor (which keeps track of what line
|
|
// and column we are on in the input data) and add the bytes to the tokenizer
|
|
// output.
|
|
//
|
|
// After the call, byte offset 0 for PeekByte() and PeekRune() will point at
|
|
// the first byte after the accepted bytes.
|
|
func (byteMode InputByteMode) AcceptMulti(bytes ...byte) {
|
|
byteMode.AddBytesToOutput(bytes...)
|
|
byteMode.MoveCursorMulti(bytes...)
|
|
}
|
|
|
|
func (byteMode InputByteMode) AddBytesToOutput(bytes ...byte) {
|
|
a := byteMode.api
|
|
if a.Output.suspended == 0 {
|
|
a.Output.AddBytes(bytes...)
|
|
}
|
|
}
|
|
|
|
// MoveCursor updates the position of the read cursor, based on the provided byte.
|
|
// This method takes newlines into account to keep track of line numbers and
|
|
// column positions for the input cursor.
|
|
//
|
|
// After the call, byte offset 0 for Peek() and PeekMulti() will point at
|
|
// the first byte at the new cursor position.
|
|
func (byteMode InputByteMode) MoveCursor(b byte) {
|
|
a := byteMode.api
|
|
if b == '\n' {
|
|
a.pointers.column = 0
|
|
a.pointers.line++
|
|
} else {
|
|
a.pointers.column++
|
|
}
|
|
|
|
a.pointers.offset++
|
|
}
|
|
|
|
// MoveCursorMulti updates the position of the read cursor, based on the provided bytes.
|
|
// This method takes newlines into account to keep track of line numbers and
|
|
// column positions for the input cursor.
|
|
//
|
|
// After the call, byte offset 0 for Peek() and PeekMulti() will point at
|
|
// the first byte at the new cursor position.
|
|
func (byteMode InputByteMode) MoveCursorMulti(bytes ...byte) {
|
|
for _, b := range bytes {
|
|
byteMode.MoveCursor(b)
|
|
}
|
|
}
|