package tokenize // InputByteMode provides byte-driven input/output functionality for the tokenize API. type InputByteMode struct { api *API } // Peek returns the byte at the provided byte offset. // // When an error occurs during reading the input, an error will be returned. // When an offset is requested that is beyond the length of the available input // data, then the error will be io.EOF. func (byteMode InputByteMode) Peek(offset int) (byte, error) { a := byteMode.api return a.reader.ByteAt(a.pointers.offset + offset) } // PeekMulti returns at max the provided maximum number of bytes at the provided // byte offset. When less bytes are available on the input, then this is not an // error as such. The returned error can in such case be set to io.EOF to indicate // that the end of the input was reached though. func (byteMode InputByteMode) PeekMulti(offset int, count int) ([]byte, error) { a := byteMode.api return a.reader.BytesAt(a.pointers.offset+offset, count) } func (byteMode InputByteMode) Accept(b byte) { a := byteMode.api if a.Output.suspended == 0 { byteMode.api.Output.AddByte(b) } byteMode.MoveCursor(b) } // AcceptMulti accepts one or more bytes that were read from the input. // This tells the tokenizer: "I've seen these bytes. I want to make use of them // for the final output, so please remember them for me. I will now continue // reading after these bytes." // // This will update the position of the cursor (which keeps track of what line // and column we are on in the input data) and add the bytes to the tokenizer // output. // // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the accepted bytes. func (byteMode InputByteMode) AcceptMulti(bytes ...byte) { a := byteMode.api if a.Output.suspended == 0 { a.Output.AddBytes(bytes...) } byteMode.MoveCursorMulti(bytes...) } // MoveCursor updates the position of the read cursor, based on the provided byte. // This method takes newlines into account to keep track of line numbers and // column positions for the input cursor. // // After the call, byte offset 0 for Peek() and PeekMulti() will point at // the first byte at the new cursor position. func (byteMode InputByteMode) MoveCursor(b byte) { a := byteMode.api if b == '\n' { a.pointers.column = 0 a.pointers.line++ } else { a.pointers.column++ } a.pointers.offset++ } // MoveCursorMulti updates the position of the read cursor, based on the provided bytes. // This method takes newlines into account to keep track of line numbers and // column positions for the input cursor. // // After the call, byte offset 0 for Peek() and PeekMulti() will point at // the first byte at the new cursor position. func (byteMode InputByteMode) MoveCursorMulti(bytes ...byte) { for _, b := range bytes { byteMode.MoveCursor(b) } }