package tokenize // ByteMode provides byte-driven input/output functionality for the tokenize API. type ByteMode struct { api *API } // Peek returns the byte at the provided byte offset. // // When an error occurs during reading the input, an error will be returned. // When an offset is requested that is beyond the length of the available input // data, then the error will be io.EOF. func (byteMode ByteMode) Peek(offset int) (byte, error) { a := byteMode.api return a.reader.ByteAt(a.stackFrame.offset + offset) } // Skip is used to skip over one or more bytes that were read from the input. // This tells the tokenizer: "I've seen these bytes. They are of no interest. // I will now continue reading after these bytes." // // This will merely update the position of the cursor (which keeps track of what // line and column we are on in the input data). The bytes are not added to // the output. // // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the skipped bytes. func (byteMode ByteMode) Skip(b byte) { f := byteMode.api.stackFrame f.moveCursorByByte(b) f.offset++ } func (byteMode ByteMode) SkipMulti(bytes ...byte) { f := byteMode.api.stackFrame for _, b := range bytes { f.moveCursorByByte(b) f.offset++ } } func (byteMode ByteMode) Accept(b byte) { byteMode.Append(b) byteMode.Skip(b) } func (byteMode ByteMode) Append(b byte) { a := byteMode.api f := a.stackFrame a.growOutputData(f.bytesEnd + 1) a.outputData[f.bytesEnd] = b f.bytesEnd++ } // AcceptMulti is used to accept one or more bytes that were read from the input. // This tells the tokenizer: "I've seen these bytes. I want to make use of them // for the final output, so please remember them for me. I will now continue // reading after these bytes." // // This will update the position of the cursor (which keeps track of what line // and column we are on in the input data) and add the bytes to the tokenizer // output. // // After the call, byte offset 0 for PeekByte() and PeekRune() will point at // the first byte after the accepted bytes. func (byteMode ByteMode) AcceptMulti(bytes ...byte) { byteMode.AppendMulti(bytes...) byteMode.SkipMulti(bytes...) } func (byteMode ByteMode) AppendMulti(bytes ...byte) { a := byteMode.api f := a.stackFrame curBytesEnd := f.bytesEnd newBytesEnd := curBytesEnd + len(bytes) a.growOutputData(newBytesEnd) copy(a.outputData[curBytesEnd:], bytes) f.bytesEnd = newBytesEnd } func (api *API) dataAddByte(b byte) { curBytesEnd := api.stackFrame.bytesEnd api.growOutputData(curBytesEnd + 1) api.outputData[curBytesEnd] = b api.stackFrame.bytesEnd++ } func (api *API) dataSetBytes(bytes ...byte) { api.Output.ClearData() api.Byte.AppendMulti(bytes...) }