OpenSourceProjects
/
go-parsekit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Working on API speed.

This commit is contained in:
Maurice Makaay 2019-07-19 14:44:44 +00:00
parent 31055a3cd3
commit 9a53ea9012
7 changed files with 572 additions and 481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
parsekit.go
View File

@ -4,7 +4,7 @@
// //
// TOKENIZE // TOKENIZE
// //
// The tokenize package's focus is to take some UTF8 input data and to produce // The tokenize package's focus is to take input data and to produce
// tokens from that input, which are bits and pieces that can be extracted // tokens from that input, which are bits and pieces that can be extracted
// from the input data and that can be recognized by the parser. // from the input data and that can be recognized by the parser.
// //
@ -12,7 +12,7 @@
// 'plus sign', 'letters') without caring at all about the actual structure // 'plus sign', 'letters') without caring at all about the actual structure
// or semantics of the input. That would be the task of the parser. // or semantics of the input. That would be the task of the parser.
// //
// I said 'traditionally', because the tokenize package implements a // I said 'traditionally', because the tokenize package provides a
// parser/combinator-style parser, which allows you to construct complex // parser/combinator-style parser, which allows you to construct complex
// tokenizers which are parsers in their own right in an easy way. // tokenizers which are parsers in their own right in an easy way.
// You can even write a tokenizer and use it in a stand-alone manner // You can even write a tokenizer and use it in a stand-alone manner
@ -36,7 +36,7 @@
// //
// When writing your own parser using parsekit, you will have to find a // When writing your own parser using parsekit, you will have to find a
// good balance between the responsibilities for the tokenizer and the parser. // good balance between the responsibilities for the tokenizer and the parser.
// The parser could provide anything from a stream of individual UTF8 runes // The parser could provide anything from a stream of individual bytes
// (where the parser will have to do all the work) to a fully parsed // (where the parser will have to do all the work) to a fully parsed
// and tokenized document for the parser to interpret. // and tokenized document for the parser to interpret.
// //

49
read/read.go
View File

@ -10,19 +10,23 @@
// The Reader can now be used to retrieve data from the input, based on their // The Reader can now be used to retrieve data from the input, based on their
// byte offset, e.g. using RuneAt(offset) or ByteAt(offset). Normally these data // byte offset, e.g. using RuneAt(offset) or ByteAt(offset). Normally these data
// will be retrieved in sequence by the user of this code, but that is not a // will be retrieved in sequence by the user of this code, but that is not a
// requirement. Let's say we retrieve the byte with offset 6 from the input // requirement. Let's say we right away ask to retrieve the byte with offset 6
// (the 'w'), then the Reader buffer be filled with runes from the io.Reader // from the input (the 'w'). The Reader buffer will then be filled with at
// until there are enough runes available to return the rune for offset 6: // least 6 bytes and the bytes at offset 6 will be returned.
//
// Note: the actual Reader would not stop after reading the byte at offset 6.
// For performance reasons, it would read as much data into the available buffer
// space as possible (but no more than the available space).
// //
// |H|e|l|l|o| |w| // |H|e|l|l|o| |w|
// 0 6 // 0 6
// //
// This means that you can retrieve data for arbitrary offsets. If you request // This means that you can retrieve data for arbitrary offsets. If you request
// an offset that is already in the Reader buffer, then the buffered data are // offsets that are already in the Reader buffer, then the buffered data are
// returned. If you request one that is not in the buffer, then the buffer will // returned. If you request an offset that is not available in the buffer, then
// be expanded. // the buffer will be expanded.
// //
// To make this into a sliding window (preserving memory space while scanning // To make this into a sliding window (which preserves memory space while scanning
// the input data), the Reader provides the method Flush(numberOfBytes). // the input data), the Reader provides the method Flush(numberOfBytes).
// This method will drop the provided number of bytes from the Reader buffer. // This method will drop the provided number of bytes from the Reader buffer.
// So when we'd do a Flush(3) on the example buffer from above, then the Reader // So when we'd do a Flush(3) on the example buffer from above, then the Reader
@ -32,7 +36,7 @@
// 0 3 // 0 3
// //
// Note that the offset for the first rune 'l' in the buffer is now 0. // Note that the offset for the first rune 'l' in the buffer is now 0.
// You can consider the input to be changed in a similar way: // You can consider the complete input to be changed in a similar way:
// //
// |l|o|,| |w|o|r|l|d|!| // |l|o|,| |w|o|r|l|d|!|
// 0 6 9 // 0 6 9
@ -94,7 +98,7 @@ type Buffer struct {
bufio *bufio.Reader // used for ReadRune() bufio *bufio.Reader // used for ReadRune()
buffer []byte // input buffer, holding runes that were read from input buffer []byte // input buffer, holding runes that were read from input
cap int // the full buffer capacity cap int // the full buffer capacity
start int // the offset from where to read buffered data in the buffer start int // the offset from where on to read buffered data in the buffer
len int // the length of the buffered data len int // the length of the buffered data
err error // a read error, if one occurred err error // a read error, if one occurred
errOffset int // the offset in the buffer at which the read error was encountered errOffset int // the offset in the buffer at which the read error was encountered
@ -102,10 +106,10 @@ type Buffer struct {
// RuneAt reads the rune at the provided byte offset. // RuneAt reads the rune at the provided byte offset.
// //
// The offset is relative to the current starting position of the Buffer. // The byte offset is relative to the current starting position of the Buffer.
// When starting reading, offset 0 will point at the start of the input. // When starting reading, offset 0 will point at the start of the input.
// After flushing, offset 0 will point at the input up to where the flush // After flushing some bytes, offset 0 will point at the input up to where
// was done. // the flush was done.
// //
// When reading was successful, the rune and the width of the rune in bytes // When reading was successful, the rune and the width of the rune in bytes
// will be returned. The returned error will be nil. // will be returned. The returned error will be nil.
@ -131,7 +135,7 @@ func (buf *Buffer) RuneAt(offset int) (rune, int, error) {
// ByteAt reads the byte at the provided byte offset. // ByteAt reads the byte at the provided byte offset.
// //
// The offset is relative to the current starting position of the Buffer. // The byte offset is relative to the current starting position of the Buffer.
// When starting reading, offset 0 will point at the start of the input. // When starting reading, offset 0 will point at the start of the input.
// After flushing, offset 0 will point at the input up to where the flush // After flushing, offset 0 will point at the input up to where the flush
// was done. // was done.
@ -140,7 +144,7 @@ func (buf *Buffer) RuneAt(offset int) (rune, int, error) {
// error will be nil. // error will be nil.
// //
// When reading failed, the byte will be 0x00 and the error will // When reading failed, the byte will be 0x00 and the error will
// be not nil. One special read fail is actually a normal situation: end // not be nil. One special read fail is actually a normal situation: end
// of file reached. In that case, the returned error wille be io.EOF. // of file reached. In that case, the returned error wille be io.EOF.
// //
// Once a read error is encountered, that same read error will guaranteed // Once a read error is encountered, that same read error will guaranteed
@ -161,7 +165,7 @@ func (buf *Buffer) fill(minBytes int) {
buf.grow(minBytes) buf.grow(minBytes)
} }
// Now we try to fill the buffer completely with data from our source. // Try to fill the buffer completely with data from our source.
// This is more efficient than only filling the data up to the point where // This is more efficient than only filling the data up to the point where
// we can read the data at the 'minBytes' position. Ideally, the buffer is // we can read the data at the 'minBytes' position. Ideally, the buffer is
// filled completely with data to work with. // filled completely with data to work with.
@ -180,13 +184,15 @@ func (buf *Buffer) fill(minBytes int) {
} }
} }
const bufferBlockSize = 1024 const defaultBufferSize = 1024
// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer. // ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("parsekit.read.Buffer: too large") var ErrTooLarge = errors.New("parsekit.read.Buffer: too large")
// grow grows the buffer to guarantee space for at least the requested amount // grow grows the buffer to guarantee space for at least the requested amount
// of bytes, either shifting data around or reallocating the buffer. // of bytes, either shifting data around or reallocating the buffer.
// When reallocating, the new buffer size will always be a multitude of the
// default buffer size.
func (buf *Buffer) grow(minBytes int) { func (buf *Buffer) grow(minBytes int) {
// When possible, grow the buffer by moving the data to the start of // When possible, grow the buffer by moving the data to the start of
// the buffer, freeing up extra capacity at the end. // the buffer, freeing up extra capacity at the end.
@ -197,9 +203,9 @@ func (buf *Buffer) grow(minBytes int) {
} }
// Grow the buffer store by allocating a new one and copying the data. // Grow the buffer store by allocating a new one and copying the data.
newbufCap := (minBytes / bufferBlockSize) * bufferBlockSize newbufCap := (minBytes / defaultBufferSize) * defaultBufferSize
if minBytes%bufferBlockSize > 0 { if minBytes%defaultBufferSize > 0 {
newbufCap += bufferBlockSize newbufCap += defaultBufferSize
} }
newStore := makeSlice(newbufCap) newStore := makeSlice(newbufCap)
copy(newStore, buf.buffer[buf.start:buf.start+buf.len]) copy(newStore, buf.buffer[buf.start:buf.start+buf.len])
@ -221,9 +227,8 @@ func makeSlice(c int) []byte {
} }
// Flush deletes the provided number of bytes from the start of the Buffer. // Flush deletes the provided number of bytes from the start of the Buffer.
// After flushing the Buffer, offset 0 as used by RuneAt() will point to // After flushing the Buffer, offset 0 as used by RuneAt() and ByteAt() will
// the rune that comes after the runes that were flushed. // point to the first byte that came after the bytes that were flushed.
// So what this basically does, is turn the Buffer into a sliding window.
func (buf *Buffer) Flush(numberOfBytes int) { func (buf *Buffer) Flush(numberOfBytes int) {
if numberOfBytes == 0 { if numberOfBytes == 0 {
return return

218
tokenize/api.go
View File

@ -7,16 +7,17 @@ import (
"git.makaay.nl/mauricem/go-parsekit/read" "git.makaay.nl/mauricem/go-parsekit/read"
) )
// API holds the internal state of a tokenizer run and provides an API that // API holds the internal state of a tokenizer run. A run uses tokenize.Handler
// tokenize.Handler functions can use to: // functions to move the tokenizer forward through the input and to provide
// tokenizer output. The API is used by these tokenize.Handler functions to:
// //
// • read and accept runes from the input (NextRune, Accept) // • access and process runes / bytes from the input data
//
// • flush processed input data that are not required anymore (FlushInput)
// //
// • fork the API for easy lookahead support (Fork, Merge, Reset, Dispose) // • fork the API for easy lookahead support (Fork, Merge, Reset, Dispose)
// //
// • flush already read input data when not needed anymore (FlushInput) // • emit tokens and/or bytes to be used by a parser
//
// • retrieve the tokenizer Result struct (Result) to read or modify the results
// //
// BASIC OPERATION: // BASIC OPERATION:
// //
@ -72,9 +73,6 @@ import (
// no bookkeeping has to be implemented when implementing a parser. // no bookkeeping has to be implemented when implementing a parser.
type API struct { type API struct {
reader *read.Buffer // the input data reader reader *read.Buffer // the input data reader
lastRune rune // the rune as retrieved by the last NextRune() call
lastRuneErr error // the error for the last NextRune() call
runeRead bool // whether or not a rune was read using NextRune()
bytes []byte // accepted bytes bytes []byte // accepted bytes
tokens []Token // accepted tokens tokens []Token // accepted tokens
stackFrames []stackFrame // the stack frames, containing stack level-specific data stackFrames []stackFrame // the stack frames, containing stack level-specific data
@ -83,9 +81,9 @@ type API struct {
} }
type stackFrame struct { type stackFrame struct {
offset int // current rune read offset relative to the Reader's sliding window offset int // the read offset (relative to the start of the reader buffer) for this stack frame
column int // The column at which the cursor is (0-indexed) column int // the column at which the cursor is (0-indexed)
line int // The line at which the cursor is (0-indexed) line int // the line at which the cursor is (0-indexed)
bytesStart int // the starting point in the API.bytes slice for runes produced by this stack level bytesStart int // the starting point in the API.bytes slice for runes produced by this stack level
bytesEnd int // the end point in the API.bytes slice for runes produced by this stack level bytesEnd int // the end point in the API.bytes slice for runes produced by this stack level
tokenStart int // the starting point in the API.tokens slice for tokens produced by this stack level tokenStart int // the starting point in the API.tokens slice for tokens produced by this stack level
@ -114,70 +112,75 @@ func NewAPI(input interface{}) *API {
return api return api
} }
// NextRune returns the rune at the current read offset. // PeekByte returns the byte at the provided byte offset.
// //
// When an invalid UTF8 rune is encountered on the input, it is replaced with // When an error occurs during reading the input, an error will be returned.
// the utf.RuneError rune. It's up to the caller to handle this as an error // When an offset is requested that is beyond the length of the available input
// when needed. // data, then the error will be io.EOF.
//
// After reading a rune it must be Accept()-ed to move the read cursor forward
// to the next rune. Doing so is mandatory. When doing a second call to NextRune()
// without explicitly accepting, this method will panic. You can see this as a
// built-in unit test, enforcing correct serialization of API method calls.
func (i *API) NextRune() (rune, error) {
if i.runeRead {
callerPanic("NextRune", "tokenize.API.{name}(): {name}() called at {caller} "+
"without a prior call to Accept()")
}
readRune, _, err := i.reader.RuneAt(i.stackFrame.offset)
i.lastRune = readRune
i.lastRuneErr = err
i.runeRead = true
return readRune, err
}
// PeekRune returns the rune at the provided offset.
//
// The read cursor and current read offset are not updated by this operation.
func (i *API) PeekRune(offset int) (rune, int, error) {
return i.reader.RuneAt(i.stackFrame.offset + offset)
}
// PeekByte returns the byte at the provided offset.
//
// The read cursor and current read offset are not updated by this operation.
func (i *API) PeekByte(offset int) (byte, error) { func (i *API) PeekByte(offset int) (byte, error) {
return i.reader.ByteAt(i.stackFrame.offset + offset) return i.reader.ByteAt(i.stackFrame.offset + offset)
} }
// Accept the last rune as read by NextRune() into the Result runes and move // SkipByte is used to skip over a single bytes that was read from the input.
// the cursor forward. // This tells the tokenizer: "I've seen this byte. It is of no interest.
// I will now continue reading after this byte."
// //
// It is not allowed to call Accept() when the previous call to NextRune() // This will merely update the position of the cursor (which keeps track of what
// returned an error. Calling Accept() in such case will result in a panic. // line and column we are on in the input data). The byte is not added to
func (i *API) Accept() { // the results.
if !i.runeRead { func (i *API) SkipByte(b byte) {
callerPanic("Accept", "tokenize.API.{name}(): {name}() called at {caller} "+ i.stackFrame.moveCursorByByte(b)
"without first calling NextRune()") i.stackFrame.offset++
} else if i.lastRuneErr != nil {
callerPanic("Accept", "tokenize.API.{name}(): {name}() called at {caller}, "+
"but the prior call to NextRune() failed")
}
i.acceptRunes(i.lastRune)
} }
func (i *API) skipBytes(bytes ...byte) { // SkipBytes is used to skip over one or more bytes that were read from the input.
// This tells the tokenizer: "I've seen these bytes. They are of no interest.
// I will now continue reading after these bytes."
//
// This will merely update the position of the cursor (which keeps track of what
// line and column we are on in the input data). The bytes are not added to
// the results.
func (i *API) SkipBytes(bytes ...byte) {
for _, b := range bytes { for _, b := range bytes {
i.stackFrame.moveCursorByByte(b) i.stackFrame.moveCursorByByte(b)
i.stackFrame.offset++ i.stackFrame.offset++
} }
i.runeRead = false
} }
func (i *API) acceptBytes(bytes ...byte) { // AcceptByte is used to accept a single byte that was read from the input.
// This tells the tokenizer: "I've seen this byte. I want to make use of it
// for the final output, so please remember it for me. I will now continue
// reading after this byte."
//
// This will update the position of the cursor (which keeps track of what line
// and column we are on in the input data) and add the byte to the tokenizer
// results.
func (i *API) AcceptByte(b byte) {
curBytesEnd := i.stackFrame.bytesEnd
maxRequiredBytes := curBytesEnd + 1
// Grow the bytes capacity when needed.
if cap(i.bytes) < maxRequiredBytes {
newBytes := make([]byte, maxRequiredBytes*2)
copy(newBytes, i.bytes)
i.bytes = newBytes
}
i.bytes[curBytesEnd] = b
i.stackFrame.moveCursorByByte(b)
i.stackFrame.bytesEnd++
i.stackFrame.offset++
}
// AcceptBytes is used to accept one or more bytes that were read from the input.
// This tells the tokenizer: "I've seen these bytes. I want to make use of them
// for the final output, so please remember them for me. I will now continue
// reading after these bytes."
//
// This will update the position of the cursor (which keeps track of what line
// and column we are on in the input data) and add the bytes to the tokenizer
// results.
func (i *API) AcceptBytes(bytes ...byte) {
curBytesEnd := i.stackFrame.bytesEnd curBytesEnd := i.stackFrame.bytesEnd
newBytesEnd := curBytesEnd + len(bytes) newBytesEnd := curBytesEnd + len(bytes)
@ -194,21 +197,88 @@ func (i *API) acceptBytes(bytes ...byte) {
i.stackFrame.offset++ i.stackFrame.offset++
} }
i.stackFrame.bytesEnd = newBytesEnd i.stackFrame.bytesEnd = newBytesEnd
i.runeRead = false
} }
func (i *API) skipRunes(width int, runes ...rune) { // PeekRune returns the UTF8 rune at the provided byte offset, including its byte width.
//
// The byte width is useful to know what byte offset you'll have to use to peek
// the next byte or rune. Some UTF8 runes take up 4 bytes of data, so when the
// first rune starts at offset = 0, the second rune might start at offset = 4.
//
// When an invalid UTF8 rune is encountered on the input, it is replaced with
// the utf.RuneError rune. It's up to the caller to handle this as an error
// when needed.
//
// When an error occurs during reading the input, an error will be returned.
// When an offset is requested that is beyond the length of the available input
// data, then the error will be io.EOF.
func (i *API) PeekRune(offset int) (rune, int, error) {
return i.reader.RuneAt(i.stackFrame.offset + offset)
}
// SkipRune is used to skip over a single rune that was read from the input.
// This tells the tokenizer: "I've seen this rune. It is of no interest.
// I will now continue reading after this rune."
//
// This will merely update the position of the cursor (which keeps track of what
// line and column we are on in the input data). The rune is not added to
// the results.
func (i *API) SkipRune(r rune) {
i.stackFrame.moveCursorByRune(r)
}
// SkipRunes is used to skip over one or more runes that were read from the input.
// This tells the tokenizer: "I've seen these runes. They are of no interest.
// I will now continue reading after these runes."
//
// This will merely update the position of the cursor (which keeps track of what
// line and column we are on in the input data). The runes are not added to
// the results.
func (i *API) SkipRunes(runes ...rune) {
for _, r := range runes { for _, r := range runes {
i.stackFrame.moveCursorByRune(r) i.stackFrame.moveCursorByRune(r)
i.stackFrame.offset += utf8.RuneLen(r)
} }
i.stackFrame.offset += width
i.runeRead = false
} }
func (i *API) acceptRunes(runes ...rune) { // AcceptRune is used to accept a single rune that was read from the input.
runesAsString := string(runes) // This tells the tokenizer: "I've seen this rune. I want to make use of it
// for the final output, so please remember it for me. I will now continue
// reading after this rune."
//
// This will update the position of the cursor (which keeps track of what line
// and column we are on in the input data) and add the rune to the tokenizer
// results.
func (i *API) AcceptRune(r rune) {
curBytesEnd := i.stackFrame.bytesEnd curBytesEnd := i.stackFrame.bytesEnd
newBytesEnd := curBytesEnd + len(runesAsString) maxRequiredBytes := curBytesEnd + utf8.UTFMax
// Grow the runes capacity when needed.
if cap(i.bytes) < maxRequiredBytes {
newBytes := make([]byte, maxRequiredBytes*2)
copy(newBytes, i.bytes)
i.bytes = newBytes
}
i.stackFrame.moveCursorByRune(r)
w := utf8.EncodeRune(i.bytes[curBytesEnd:], r)
i.stackFrame.bytesEnd += w
i.stackFrame.offset += w
}
// AcceptRunes is used to accept one or more runes that were read from the input.
// This tells the tokenizer: "I've seen these runes. I want to make use of them
// for the final output, so please remember them for me. I will now continue
// reading after these runes."
//
// This will update the position of the cursor (which keeps track of what line
// and column we are on in the input data) and add the runes to the tokenizer
// results.
func (i *API) AcceptRunes(runes ...rune) {
runesAsString := string(runes)
byteLen := len(runesAsString)
curBytesEnd := i.stackFrame.bytesEnd
newBytesEnd := curBytesEnd + byteLen
// Grow the runes capacity when needed. // Grow the runes capacity when needed.
if cap(i.bytes) < newBytesEnd { if cap(i.bytes) < newBytesEnd {
@ -223,12 +293,12 @@ func (i *API) acceptRunes(runes ...rune) {
copy(i.bytes[curBytesEnd:], runesAsString) copy(i.bytes[curBytesEnd:], runesAsString)
i.stackFrame.bytesEnd = newBytesEnd i.stackFrame.bytesEnd = newBytesEnd
i.stackFrame.offset += len(runesAsString) i.stackFrame.offset += byteLen
i.runeRead = false
} }
// Fork forks off a child of the API struct. It will reuse the same // Fork forks off a child of the API struct. It will reuse the same
// read buffer and cursor position, but for the rest this is a fresh API. // read buffer and cursor position, but for the rest this can be considered
// a fresh API.
// //
// By forking an API, you can freely work with the forked child, without // By forking an API, you can freely work with the forked child, without
// affecting the parent API. This is for example useful when you must perform // affecting the parent API. This is for example useful when you must perform
@ -256,7 +326,6 @@ func (i *API) Fork() int {
} }
i.stackLevel++ i.stackLevel++
i.runeRead = false
// This can be written in a shorter way, but this turned out to // This can be written in a shorter way, but this turned out to
// be the best way performance-wise. // be the best way performance-wise.
@ -318,7 +387,6 @@ func (i *API) Merge(stackLevel int) {
parent.column = i.stackFrame.column parent.column = i.stackFrame.column
i.stackFrame.err = nil i.stackFrame.err = nil
i.runeRead = false
} }
func (i *API) Dispose(stackLevel int) { func (i *API) Dispose(stackLevel int) {
@ -332,13 +400,11 @@ func (i *API) Dispose(stackLevel int) {
"(forgot to Dispose() a forked child?)", stackLevel, i.stackLevel) "(forgot to Dispose() a forked child?)", stackLevel, i.stackLevel)
} }
i.runeRead = false
i.stackLevel = stackLevel - 1 i.stackLevel = stackLevel - 1
i.stackFrame = &i.stackFrames[stackLevel-1] i.stackFrame = &i.stackFrames[stackLevel-1]
} }
func (i *API) Reset() { func (i *API) Reset() {
i.runeRead = false
if i.stackLevel == 0 { if i.stackLevel == 0 {
i.stackFrame.column = 0 i.stackFrame.column = 0
i.stackFrame.line = 0 i.stackFrame.line = 0

312
tokenize/api_test.go
View File

@ -13,18 +13,18 @@ func ExampleNewAPI() {
// Output: // Output:
} }
func ExampleAPI_NextRune() { // func ExampleAPI_NextRune() {
api := tokenize.NewAPI("The input that the API will handle") // api := tokenize.NewAPI("The input that the API will handle")
r, err := api.NextRune() // r, err := api.NextRune()
fmt.Printf("Rune read from input; %c\n", r) // fmt.Printf("Rune read from input; %c\n", r)
fmt.Printf("The error: %v\n", err) // fmt.Printf("The error: %v\n", err)
fmt.Printf("API results: %q\n", api.String()) // fmt.Printf("API results: %q\n", api.String())
// Output: // // Output:
// Rune read from input; T // // Rune read from input; T
// The error: <nil> // // The error: <nil>
// API results: "" // // API results: ""
} // }
func ExampleAPI_PeekRune() { func ExampleAPI_PeekRune() {
api := tokenize.NewAPI("The input that the API will handle") api := tokenize.NewAPI("The input that the API will handle")
@ -40,13 +40,19 @@ func ExampleAPI_PeekRune() {
// API EOF // API EOF
} }
func ExampleAPI_Accept() { func ExampleAPI_AcceptRune() {
api := tokenize.NewAPI("The input that the API will handle") api := tokenize.NewAPI("The input that the API will handle")
api.NextRune() // reads 'T'
api.Accept() // adds 'T' to the API results // reads 'T' and adds it to the API results
api.NextRune() // reads 'h' r, _, _ := api.PeekRune(0)
api.Accept() // adds 'h' to the API results api.AcceptRune(r)
api.NextRune() // reads 'e', but it is not added to the API results
// reads 'h' and adds it to the API results
r, _, _ = api.PeekRune(0)
api.AcceptRune(r)
// reads 'e', but does not add it to the API results
r, _, _ = api.PeekRune(0)
fmt.Printf("API results: %q\n", api.String()) fmt.Printf("API results: %q\n", api.String())
@ -91,31 +97,32 @@ func ExampleAPI_modifyingResults() {
// API second result token: 73("Zaphod") // API second result token: 73("Zaphod")
} }
func ExampleAPI_Reset() { // TODO FIXME
api := tokenize.NewAPI("Very important input!") // func ExampleAPI_Reset() {
// api := tokenize.NewAPI("Very important input!")
api.NextRune() // read 'V' // api.NextRune() // read 'V'
api.Accept() // api.Accept()
api.NextRune() // read 'e' // api.NextRune() // read 'e'
api.Accept() // api.Accept()
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) // fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor())
// Reset clears the results. // // Reset clears the results.
api.Reset() // api.Reset()
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) // fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor())
// So then doing the same read operations, the same data are read. // // So then doing the same read operations, the same data are read.
api.NextRune() // read 'V' // api.NextRune() // read 'V'
api.Accept() // api.Accept()
api.NextRune() // read 'e' // api.NextRune() // read 'e'
api.Accept() // api.Accept()
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) // fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor())
// Output: // // Output:
// API results: "Ve" at line 1, column 3 // // API results: "Ve" at line 1, column 3
// API results: "" at start of file // // API results: "" at start of file
// API results: "Ve" at line 1, column 3 // // API results: "Ve" at line 1, column 3
} // }
func ExampleAPI_Fork() { func ExampleAPI_Fork() {
// This custom Handler checks for input 'a', 'b' or 'c'. // This custom Handler checks for input 'a', 'b' or 'c'.
@ -157,146 +164,149 @@ func ExampleAPI_Fork() {
// <nil> mismatch at start of file // <nil> mismatch at start of file
} }
func ExampleAPI_Merge() { // TODO FIXME
tokenHandler := func(t *tokenize.API) bool { // func ExampleAPI_Merge() {
child1 := t.Fork() // tokenHandler := func(t *tokenize.API) bool {
t.NextRune() // reads 'H' // child1 := t.Fork()
t.Accept() // t.NextRune() // reads 'H'
t.NextRune() // reads 'i' // t.Accept()
t.Accept() // t.NextRune() // reads 'i'
// t.Accept()
child2 := t.Fork() // child2 := t.Fork()
t.NextRune() // reads ' ' // t.NextRune() // reads ' '
t.Accept() // t.Accept()
t.NextRune() // reads 'm' // t.NextRune() // reads 'm'
t.Accept() // t.Accept()
t.Dispose(child2) // t.Dispose(child2)
t.Merge(child1) // We merge child1, which has read 'H' and 'i' only. // t.Merge(child1) // We merge child1, which has read 'H' and 'i' only.
t.Dispose(child1) // and clean up child1 to return to the parent // t.Dispose(child1) // and clean up child1 to return to the parent
return true // return true
} // }
result, _ := tokenize.New(tokenHandler)("Hi mister X!") // result, _ := tokenize.New(tokenHandler)("Hi mister X!")
fmt.Println(result.String()) // fmt.Println(result.String())
// Output: // // Output:
// Hi // // Hi
} // }
func TestMultipleLevelsOfForksAndMerges(t *testing.T) { // TODO FIXME
api := tokenize.NewAPI("abcdefghijklmnopqrstuvwxyz") // func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
// api := tokenize.NewAPI("abcdefghijklmnopqrstuvwxyz")
// Fork a few levels. // // Fork a few levels.
child1 := api.Fork() // child1 := api.Fork()
child2 := api.Fork() // child2 := api.Fork()
child3 := api.Fork() // child3 := api.Fork()
child4 := api.Fork() // child4 := api.Fork()
// Read a rune 'a' from child4. // // Read a rune 'a' from child4.
r, _ := api.NextRune() // r, _ := api.NextRune()
AssertEqual(t, 'a', r, "child4 rune 1") // AssertEqual(t, 'a', r, "child4 rune 1")
api.Accept() // api.Accept()
AssertEqual(t, "a", api.String(), "child4 runes after rune 1") // AssertEqual(t, "a", api.String(), "child4 runes after rune 1")
// Read another rune 'b' from child4. // // Read another rune 'b' from child4.
r, _ = api.NextRune() // r, _ = api.NextRune()
AssertEqual(t, 'b', r, "child4 rune 2") // AssertEqual(t, 'b', r, "child4 rune 2")
api.Accept() // api.Accept()
AssertEqual(t, "ab", api.String(), "child4 runes after rune 2") // AssertEqual(t, "ab", api.String(), "child4 runes after rune 2")
// Merge "ab" from child4 to child3. // // Merge "ab" from child4 to child3.
api.Merge(child4) // api.Merge(child4)
AssertEqual(t, "", api.String(), "child4 runes after first merge") // AssertEqual(t, "", api.String(), "child4 runes after first merge")
// Read some more from child4. // // Read some more from child4.
r, _ = api.NextRune() // r, _ = api.NextRune()
AssertEqual(t, 'c', r, "child4 rune 3") // AssertEqual(t, 'c', r, "child4 rune 3")
api.Accept() // api.Accept()
AssertEqual(t, "c", api.String(), "child4 runes after rune 1") // AssertEqual(t, "c", api.String(), "child4 runes after rune 1")
AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child4 rune 3") // AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child4 rune 3")
// Merge "c" from child4 to child3. // // Merge "c" from child4 to child3.
api.Merge(child4) // api.Merge(child4)
// And dispose of child4, making child3 the active stack level. // // And dispose of child4, making child3 the active stack level.
api.Dispose(child4) // api.Dispose(child4)
// Child3 should now have the compbined results "abc" from child4's work. // // Child3 should now have the compbined results "abc" from child4's work.
AssertEqual(t, "abc", api.String(), "child3 after merge of child4") // AssertEqual(t, "abc", api.String(), "child3 after merge of child4")
AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child3 rune 3, after merge of child4") // AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child3 rune 3, after merge of child4")
// Now read some data from child3. // // Now read some data from child3.
r, _ = api.NextRune() // r, _ = api.NextRune()
AssertEqual(t, 'd', r, "child3 rune 5") // AssertEqual(t, 'd', r, "child3 rune 5")
api.Accept() // api.Accept()
r, _ = api.NextRune() // r, _ = api.NextRune()
AssertEqual(t, 'e', r, "child3 rune 5") // AssertEqual(t, 'e', r, "child3 rune 5")
api.Accept() // api.Accept()
r, _ = api.NextRune() // r, _ = api.NextRune()
AssertEqual(t, 'f', r, "child3 rune 5") // AssertEqual(t, 'f', r, "child3 rune 5")
api.Accept() // api.Accept()
AssertEqual(t, "abcdef", api.String(), "child3 total result after rune 6") // AssertEqual(t, "abcdef", api.String(), "child3 total result after rune 6")
// Temporarily go some new forks from here, but don't use their outcome. // // Temporarily go some new forks from here, but don't use their outcome.
child3sub1 := api.Fork() // child3sub1 := api.Fork()
api.NextRune() // api.NextRune()
api.Accept() // api.Accept()
api.NextRune() // api.NextRune()
api.Accept() // api.Accept()
child3sub2 := api.Fork() // child3sub2 := api.Fork()
api.NextRune() // api.NextRune()
api.Accept() // api.Accept()
api.Merge(child3sub2) // do merge sub2 down to sub1 // api.Merge(child3sub2) // do merge sub2 down to sub1
api.Dispose(child3sub2) // and dispose of sub2 // api.Dispose(child3sub2) // and dispose of sub2
api.Dispose(child3sub1) // but dispose of sub1 without merging // api.Dispose(child3sub1) // but dispose of sub1 without merging
// Instead merge the results from before this forking segway from child3 to child2 // // Instead merge the results from before this forking segway from child3 to child2
// and dispose of it. // // and dispose of it.
api.Merge(child3) // api.Merge(child3)
api.Dispose(child3) // api.Dispose(child3)
AssertEqual(t, "abcdef", api.String(), "child2 total result after merge of child3") // AssertEqual(t, "abcdef", api.String(), "child2 total result after merge of child3")
AssertEqual(t, "line 1, column 7", api.Cursor(), "cursor child2 after merge child3") // AssertEqual(t, "line 1, column 7", api.Cursor(), "cursor child2 after merge child3")
// Merge child2 to child1 and dispose of it. // // Merge child2 to child1 and dispose of it.
api.Merge(child2) // api.Merge(child2)
api.Dispose(child2) // api.Dispose(child2)
// Merge child1 a few times to the top level api. // // Merge child1 a few times to the top level api.
api.Merge(child1) // api.Merge(child1)
api.Merge(child1) // api.Merge(child1)
api.Merge(child1) // api.Merge(child1)
api.Merge(child1) // api.Merge(child1)
// And dispose of it. // // And dispose of it.
api.Dispose(child1) // api.Dispose(child1)
// Read some data from the top level api. // // Read some data from the top level api.
r, _ = api.NextRune() // r, _ = api.NextRune()
api.Accept() // api.Accept()
AssertEqual(t, "abcdefg", api.String(), "api string end result") // AssertEqual(t, "abcdefg", api.String(), "api string end result")
AssertEqual(t, "line 1, column 8", api.Cursor(), "api cursor end result") // AssertEqual(t, "line 1, column 8", api.Cursor(), "api cursor end result")
} // }
func TestClearRunes(t *testing.T) { // TODO FIXME
api := tokenize.NewAPI("Laphroaig") // func TestClearRunes(t *testing.T) {
api.NextRune() // Read 'L' // api := tokenize.NewAPI("Laphroaig")
api.Accept() // Add to runes // api.NextRune() // Read 'L'
api.NextRune() // Read 'a' // api.Accept() // Add to runes
api.Accept() // Add to runes // api.NextRune() // Read 'a'
api.ClearRunes() // Clear the runes, giving us a fresh start. // api.Accept() // Add to runes
api.NextRune() // Read 'p' // api.ClearRunes() // Clear the runes, giving us a fresh start.
api.Accept() // Add to runes // api.NextRune() // Read 'p'
api.NextRune() // Read 'r' // api.Accept() // Add to runes
api.Accept() // Add to runes // api.NextRune() // Read 'r'
// api.Accept() // Add to runes
AssertEqual(t, "ph", api.String(), "api string end result") // AssertEqual(t, "ph", api.String(), "api string end result")
} // }
func TestMergeScenariosForTokens(t *testing.T) { func TestMergeScenariosForTokens(t *testing.T) {
api := tokenize.NewAPI("") api := tokenize.NewAPI("")

106
tokenize/handlers_builtin.go
View File

@ -352,7 +352,7 @@ func MatchByte(expected byte) Handler {
return func(t *API) bool { return func(t *API) bool {
b, err := t.PeekByte(0) b, err := t.PeekByte(0)
if err == nil && b == expected { if err == nil && b == expected {
t.acceptBytes(b) t.AcceptBytes(b)
return true return true
} }
return false return false
@ -367,7 +367,7 @@ func MatchRune(expected rune) Handler {
return func(t *API) bool { return func(t *API) bool {
r, _, err := t.PeekRune(0) r, _, err := t.PeekRune(0)
if err == nil && r == expected { if err == nil && r == expected {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
return false return false
@ -384,7 +384,7 @@ func MatchBytes(expected ...byte) Handler {
} }
for _, e := range expected { for _, e := range expected {
if b == e { if b == e {
t.acceptBytes(b) t.AcceptBytes(b)
return true return true
} }
} }
@ -414,7 +414,7 @@ func MatchRunes(expected ...rune) Handler {
} }
for _, e := range expected { for _, e := range expected {
if r == e { if r == e {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
} }
@ -436,7 +436,7 @@ func MatchByteRange(start byte, end byte) Handler {
return func(t *API) bool { return func(t *API) bool {
r, err := t.PeekByte(0) r, err := t.PeekByte(0)
if err == nil && r >= start && r <= end { if err == nil && r >= start && r <= end {
t.acceptBytes(r) t.AcceptBytes(r)
return true return true
} }
return false return false
@ -460,7 +460,7 @@ func MatchRuneRange(start rune, end rune) Handler {
return func(t *API) bool { return func(t *API) bool {
r, _, err := t.PeekRune(0) r, _, err := t.PeekRune(0)
if err == nil && r >= start && r <= end { if err == nil && r >= start && r <= end {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
return false return false
@ -476,13 +476,13 @@ func MatchNewline() Handler {
return false return false
} }
if b1 == '\n' { if b1 == '\n' {
t.acceptBytes(b1) t.AcceptBytes(b1)
return true return true
} }
if b1 == '\r' { if b1 == '\r' {
b2, err := t.PeekByte(1) b2, err := t.PeekByte(1)
if err == nil && b2 == '\n' { if err == nil && b2 == '\n' {
t.acceptBytes(b1, b2) t.AcceptBytes(b1, b2)
return true return true
} }
} }
@ -499,7 +499,7 @@ func MatchBlank() Handler {
return func(t *API) bool { return func(t *API) bool {
b, err := t.PeekByte(0) b, err := t.PeekByte(0)
if err == nil && (b == ' ' || b == '\t') { if err == nil && (b == ' ' || b == '\t') {
t.acceptBytes(b) t.AcceptBytes(b)
return true return true
} }
return false return false
@ -520,7 +520,7 @@ func MatchBlanks() Handler {
if err != nil || (b != ' ' && b != '\t') { if err != nil || (b != ' ' && b != '\t') {
return false return false
} }
t.acceptBytes(b) t.AcceptBytes(b)
// Now match any number of followup blanks. We've already got // Now match any number of followup blanks. We've already got
// a successful match at this point, so we'll always return true at the end. // a successful match at this point, so we'll always return true at the end.
@ -529,7 +529,7 @@ func MatchBlanks() Handler {
if err != nil || (b != ' ' && b != '\t') { if err != nil || (b != ' ' && b != '\t') {
return true return true
} }
t.acceptBytes(b) t.AcceptBytes(b)
} }
} }
} }
@ -549,9 +549,9 @@ func MatchWhitespace() Handler {
if err != nil || b2 != '\n' { if err != nil || b2 != '\n' {
return false return false
} }
t.acceptBytes(b1, b2) t.AcceptBytes(b1, b2)
} else { } else {
t.acceptBytes(b1) t.AcceptBytes(b1)
} }
// Now match any number of followup whitespace. We've already got // Now match any number of followup whitespace. We've already got
@ -566,9 +566,9 @@ func MatchWhitespace() Handler {
if err != nil || b2 != '\n' { if err != nil || b2 != '\n' {
return true return true
} }
t.acceptBytes(b1, b2) t.AcceptBytes(b1, b2)
} else { } else {
t.acceptBytes(b1) t.AcceptBytes(b1)
} }
} }
} }
@ -590,7 +590,7 @@ func MatchByteByCallback(callback func(byte) bool) Handler {
return func(t *API) bool { return func(t *API) bool {
b, err := t.PeekByte(0) b, err := t.PeekByte(0)
if err == nil && callback(b) { if err == nil && callback(b) {
t.acceptBytes(b) t.AcceptBytes(b)
return true return true
} }
return false return false
@ -607,7 +607,7 @@ func MatchRuneByCallback(callback func(rune) bool) Handler {
return func(t *API) bool { return func(t *API) bool {
r, _, err := t.PeekRune(0) r, _, err := t.PeekRune(0)
if err == nil && callback(r) { if err == nil && callback(r) {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
return false return false
@ -622,13 +622,13 @@ func MatchEndOfLine() Handler {
return err == io.EOF return err == io.EOF
} }
if b1 == '\n' { if b1 == '\n' {
t.acceptBytes(b1) t.AcceptBytes(b1)
return true return true
} }
if b1 == '\r' { if b1 == '\r' {
b2, _ := t.PeekByte(1) b2, _ := t.PeekByte(1)
if b2 == '\n' { if b2 == '\n' {
t.acceptBytes(b1, b2) t.AcceptBytes(b1, b2)
return true return true
} }
} }
@ -657,7 +657,7 @@ func MatchStr(expected string) Handler {
offset += w offset += w
} }
} }
t.acceptRunes(expectedRunes...) t.AcceptRunes(expectedRunes...)
return true return true
} }
} }
@ -689,7 +689,7 @@ func MatchStrNoCase(expected string) Handler {
} }
i++ i++
} }
t.acceptRunes(matches...) t.AcceptRunes(matches...)
return true return true
} }
} }
@ -763,7 +763,7 @@ func MatchNot(handler Handler) Handler {
t.Dispose(child) t.Dispose(child)
r, _, err := t.PeekRune(0) r, _, err := t.PeekRune(0)
if err == nil { if err == nil {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
return false return false
@ -961,7 +961,7 @@ func MatchSigned(handler Handler) Handler {
return false return false
} }
if b == '-' || b == '+' { if b == '-' || b == '+' {
t.acceptBytes(b) t.AcceptBytes(b)
} }
if handler(t) { if handler(t) {
t.Merge(child) t.Merge(child)
@ -1019,7 +1019,7 @@ func MatchAnyByte() Handler {
return func(t *API) bool { return func(t *API) bool {
b, err := t.PeekByte(0) b, err := t.PeekByte(0)
if err == nil { if err == nil {
t.acceptBytes(b) t.AcceptBytes(b)
return true return true
} }
return false return false
@ -1033,7 +1033,7 @@ func MatchAnyRune() Handler {
return func(t *API) bool { return func(t *API) bool {
r, _, err := t.PeekRune(0) r, _, err := t.PeekRune(0)
if err == nil { if err == nil {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
return false return false
@ -1046,7 +1046,7 @@ func MatchValidRune() Handler {
return func(t *API) bool { return func(t *API) bool {
r, _, err := t.PeekRune(0) r, _, err := t.PeekRune(0)
if err == nil && r != utf8.RuneError { if err == nil && r != utf8.RuneError {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
return false return false
@ -1059,7 +1059,7 @@ func MatchInvalidRune() Handler {
return func(t *API) bool { return func(t *API) bool {
r, _, err := t.PeekRune(0) r, _, err := t.PeekRune(0)
if err == nil && r == utf8.RuneError { if err == nil && r == utf8.RuneError {
t.acceptRunes(r) t.AcceptRunes(r)
return true return true
} }
return false return false
@ -1081,7 +1081,7 @@ func MatchDigits() Handler {
if err != nil || b < '0' || b > '9' { if err != nil || b < '0' || b > '9' {
return false return false
} }
t.acceptBytes(b) t.AcceptBytes(b)
// Continue accepting bytes as long as they are digits. // Continue accepting bytes as long as they are digits.
for { for {
@ -1089,7 +1089,7 @@ func MatchDigits() Handler {
if err != nil || b < '0' || b > '9' { if err != nil || b < '0' || b > '9' {
return true return true
} }
t.acceptBytes(b) t.AcceptBytes(b)
} }
} }
} }
@ -1120,18 +1120,18 @@ func MatchInteger(normalize bool) Handler {
// The next character is a zero, skip the leading zero and check again. // The next character is a zero, skip the leading zero and check again.
if err == nil && b2 == b { if err == nil && b2 == b {
t.skipBytes('0') t.SkipBytes('0')
continue continue
} }
// The next character is not a zero, nor a digit at all. // The next character is not a zero, nor a digit at all.
// We're looking at a zero on its own here. // We're looking at a zero on its own here.
if err != nil || b2 < '1' || b2 > '9' { if err != nil || b2 < '1' || b2 > '9' {
t.acceptBytes('0') t.AcceptBytes('0')
return true return true
} }
// The next character is a digit. SKip the leading zero and go with the digit. // The next character is a digit. SKip the leading zero and go with the digit.
t.skipBytes('0') t.SkipBytes('0')
t.acceptBytes(b2) t.AcceptBytes(b2)
break break
} }
} }
@ -1142,7 +1142,7 @@ func MatchInteger(normalize bool) Handler {
if err != nil || b < '0' || b > '9' { if err != nil || b < '0' || b > '9' {
return true return true
} }
t.acceptBytes(b) t.AcceptBytes(b)
} }
} }
} }
@ -1169,24 +1169,24 @@ func MatchDecimal(normalize bool) Handler {
// The next character is a zero, skip the leading zero and check again. // The next character is a zero, skip the leading zero and check again.
if err == nil && b2 == b { if err == nil && b2 == b {
t.skipBytes('0') t.SkipBytes('0')
continue continue
} }
// The next character is a dot, go with the zero before the dot and // The next character is a dot, go with the zero before the dot and
// let the upcoming code handle the dot. // let the upcoming code handle the dot.
if err == nil && b2 == '.' { if err == nil && b2 == '.' {
t.acceptBytes('0') t.AcceptBytes('0')
break break
} }
// The next character is not a zero, nor a digit at all. // The next character is not a zero, nor a digit at all.
// We're looking at a zero on its own here. // We're looking at a zero on its own here.
if err != nil || b2 < '1' || b2 > '9' { if err != nil || b2 < '1' || b2 > '9' {
t.acceptBytes('0') t.AcceptBytes('0')
return true return true
} }
// The next character is a digit. SKip the leading zero and go with the digit. // The next character is a digit. SKip the leading zero and go with the digit.
t.skipBytes('0') t.SkipBytes('0')
t.acceptBytes(b2) t.AcceptBytes(b2)
break break
} }
} }
@ -1197,7 +1197,7 @@ func MatchDecimal(normalize bool) Handler {
if err != nil || b < '0' || b > '9' { if err != nil || b < '0' || b > '9' {
break break
} }
t.acceptBytes(b) t.AcceptBytes(b)
} }
// No dot or no digit after a dot? Then we're done. // No dot or no digit after a dot? Then we're done.
@ -1210,13 +1210,13 @@ func MatchDecimal(normalize bool) Handler {
} }
// Continue accepting bytes as long as they are digits. // Continue accepting bytes as long as they are digits.
t.acceptBytes('.', b) t.AcceptBytes('.', b)
for { for {
b, err = t.PeekByte(0) b, err = t.PeekByte(0)
if err != nil || b < '0' || b > '9' { if err != nil || b < '0' || b > '9' {
break break
} }
t.acceptBytes(b) t.AcceptBytes(b)
} }
return true return true
} }
@ -1236,47 +1236,47 @@ func MatchBoolean() Handler {
return false return false
} }
if b1 == '1' || b1 == '0' { if b1 == '1' || b1 == '0' {
t.acceptBytes(b1) t.AcceptBytes(b1)
return true return true
} }
if b1 == 't' || b1 == 'T' { if b1 == 't' || b1 == 'T' {
b2, err := t.PeekByte(1) b2, err := t.PeekByte(1)
if err != nil || (b2 != 'R' && b2 != 'r') { if err != nil || (b2 != 'R' && b2 != 'r') {
t.acceptBytes(b1) t.AcceptBytes(b1)
return true return true
} }
b3, _ := t.PeekByte(2) b3, _ := t.PeekByte(2)
b4, err := t.PeekByte(3) b4, err := t.PeekByte(3)
if err == nil && b2 == 'r' && b3 == 'u' && b4 == 'e' { if err == nil && b2 == 'r' && b3 == 'u' && b4 == 'e' {
t.acceptBytes(b1, b2, b3, b4) t.AcceptBytes(b1, b2, b3, b4)
return true return true
} }
if err == nil && b1 == 'T' && b2 == 'R' && b3 == 'U' && b4 == 'E' { if err == nil && b1 == 'T' && b2 == 'R' && b3 == 'U' && b4 == 'E' {
t.acceptBytes(b1, b2, b3, b4) t.AcceptBytes(b1, b2, b3, b4)
return true return true
} }
t.acceptBytes(b1) t.AcceptBytes(b1)
return true return true
} }
if b1 == 'f' || b1 == 'F' { if b1 == 'f' || b1 == 'F' {
b2, err := t.PeekByte(1) b2, err := t.PeekByte(1)
if err != nil || (b2 != 'A' && b2 != 'a') { if err != nil || (b2 != 'A' && b2 != 'a') {
t.acceptBytes(b1) t.AcceptBytes(b1)
return true return true
} }
b3, _ := t.PeekByte(2) b3, _ := t.PeekByte(2)
b4, _ := t.PeekByte(3) b4, _ := t.PeekByte(3)
b5, err := t.PeekByte(4) b5, err := t.PeekByte(4)
if err == nil && b2 == 'a' && b3 == 'l' && b4 == 's' && b5 == 'e' { if err == nil && b2 == 'a' && b3 == 'l' && b4 == 's' && b5 == 'e' {
t.acceptBytes(b1, b2, b3, b4, b5) t.AcceptBytes(b1, b2, b3, b4, b5)
return true return true
} }
if err == nil && b1 == 'F' && b2 == 'A' && b3 == 'L' && b4 == 'S' && b5 == 'E' { if err == nil && b1 == 'F' && b2 == 'A' && b3 == 'L' && b4 == 'S' && b5 == 'E' {
t.acceptBytes(b1, b2, b3, b4, b5) t.AcceptBytes(b1, b2, b3, b4, b5)
return true return true
} }
t.acceptBytes(b1) t.AcceptBytes(b1)
return true return true
} }
return false return false
@ -1325,7 +1325,7 @@ func MatchHexDigit() Handler {
return func(t *API) bool { return func(t *API) bool {
b, err := t.PeekByte(0) b, err := t.PeekByte(0)
if err == nil && ((b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F')) { if err == nil && ((b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F')) {
t.acceptBytes(b) t.AcceptBytes(b)
return true return true
} }
return false return false
@ -1567,7 +1567,7 @@ func ModifyDropUntilEndOfLine() Handler {
if b == '\n' { if b == '\n' {
return true return true
} }
t.skipBytes(b) t.SkipBytes(b)
} }
} }
} }

206
tokenize/tokenizer_test.go
View File

@ -2,10 +2,7 @@ package tokenize_test
import ( import (
"fmt" "fmt"
"io"
"strings"
"testing" "testing"
"unicode/utf8"
tokenize "git.makaay.nl/mauricem/go-parsekit/tokenize" tokenize "git.makaay.nl/mauricem/go-parsekit/tokenize"
) )
@ -53,58 +50,63 @@ func ExampleNew() {
// Error: mismatch at start of file // Error: mismatch at start of file
} }
func TestCallingNextRune_ReturnsNextRune(t *testing.T) { // TODO FIXME
api := makeTokenizeAPI() // func TestCallingNextRune_ReturnsNextRune(t *testing.T) {
r, _ := api.NextRune() // api := makeTokenizeAPI()
AssertEqual(t, 'T', r, "first rune") // r, _ := api.NextRune()
} // AssertEqual(t, 'T', r, "first rune")
// }
func TestInputCanAcceptRunesFromReader(t *testing.T) { // TODO FIXME
i := makeTokenizeAPI() // func TestInputCanAcceptRunesFromReader(t *testing.T) {
i.NextRune() // i := makeTokenizeAPI()
i.Accept() // i.NextRune()
i.NextRune() // i.Accept()
i.Accept() // i.NextRune()
i.NextRune() // i.Accept()
i.Accept() // i.NextRune()
AssertEqual(t, "Tes", i.String(), "i.String()") // i.Accept()
} // AssertEqual(t, "Tes", i.String(), "i.String()")
// }
func TestCallingNextRuneTwice_Panics(t *testing.T) { // TODO FIXME
AssertPanic(t, PanicT{ // func TestCallingNextRuneTwice_Panics(t *testing.T) {
Function: func() { // AssertPanic(t, PanicT{
i := makeTokenizeAPI() // Function: func() {
i.NextRune() // i := makeTokenizeAPI()
i.NextRune() // i.NextRune()
}, // i.NextRune()
Regexp: true, // },
Expect: `tokenize\.API\.NextRune\(\): NextRune\(\) called at /.*_test\.go:\d+ ` + // Regexp: true,
`without a prior call to Accept\(\)`, // Expect: `tokenize\.API\.NextRune\(\): NextRune\(\) called at /.*_test\.go:\d+ ` +
}) // `without a prior call to Accept\(\)`,
} // })
// }
func TestCallingAcceptWithoutCallingNextRune_Panics(t *testing.T) { // TODO FIXME
api := makeTokenizeAPI() // func TestCallingAcceptWithoutCallingNextRune_Panics(t *testing.T) {
AssertPanic(t, PanicT{ // api := makeTokenizeAPI()
Function: api.Accept, // AssertPanic(t, PanicT{
Regexp: true, // Function: api.Accept,
Expect: `tokenize\.API\.Accept\(\): Accept\(\) called at /.*test\.go:\d+ ` + // Regexp: true,
`without first calling NextRune\(\)`, // Expect: `tokenize\.API\.Accept\(\): Accept\(\) called at /.*test\.go:\d+ ` +
}) // `without first calling NextRune\(\)`,
} // })
// }
func TestCallingAcceptAfterReadError_Panics(t *testing.T) { // TODO FIXME
api := tokenize.NewAPI("") // func TestCallingAcceptAfterReadError_Panics(t *testing.T) {
AssertPanic(t, PanicT{ // api := tokenize.NewAPI("")
Function: func() { // AssertPanic(t, PanicT{
api.NextRune() // Function: func() {
api.Accept() // api.NextRune()
}, // api.Accept()
Regexp: true, // },
Expect: `tokenize\.API\.Accept\(\): Accept\(\) called at /.*_test\.go:\d+` + // Regexp: true,
`, but the prior call to NextRune\(\) failed`, // Expect: `tokenize\.API\.Accept\(\): Accept\(\) called at /.*_test\.go:\d+` +
}) // `, but the prior call to NextRune\(\) failed`,
} // })
// }
func TestCallingMergeOnTopLevelAPI_Panics(t *testing.T) { func TestCallingMergeOnTopLevelAPI_Panics(t *testing.T) {
AssertPanic(t, PanicT{ AssertPanic(t, PanicT{
@ -166,57 +168,61 @@ func TestCallingForkOnForkedParentAPI_Panics(t *testing.T) {
`on API stack level 2, but the current stack level is 3 \(forgot to Dispose\(\) a forked child\?\)`}) `on API stack level 2, but the current stack level is 3 \(forgot to Dispose\(\) a forked child\?\)`})
} }
func TestForkingInput_ClearsLastRune(t *testing.T) { // TODO FIXME
AssertPanic(t, PanicT{ // func TestForkingInput_ClearsLastRune(t *testing.T) {
Function: func() { // AssertPanic(t, PanicT{
i := makeTokenizeAPI() // Function: func() {
i.NextRune() // i := makeTokenizeAPI()
i.Fork() // i.NextRune()
i.Accept() // i.Fork()
}, // i.Accept()
Regexp: true, // },
Expect: `tokenize\.API\.Accept\(\): Accept\(\) called at /.*_test\.go:\d+ without first calling NextRune\(\)`, // Regexp: true,
}) // Expect: `tokenize\.API\.Accept\(\): Accept\(\) called at /.*_test\.go:\d+ without first calling NextRune\(\)`,
} // })
// }
func TestAccept_UpdatesCursor(t *testing.T) { // TODO FIXME
i := tokenize.NewAPI(strings.NewReader("input\r\nwith\r\nnewlines")) // func TestAccept_UpdatesCursor(t *testing.T) {
AssertEqual(t, "start of file", i.Cursor(), "cursor 1") // i := tokenize.NewAPI(strings.NewReader("input\r\nwith\r\nnewlines"))
for j := 0; j < 6; j++ { // read "input\r", cursor end up at "\n" // AssertEqual(t, "start of file", i.Cursor(), "cursor 1")
i.NextRune() // for j := 0; j < 6; j++ { // read "input\r", cursor end up at "\n"
i.Accept() // i.NextRune()
} // i.Accept()
AssertEqual(t, "line 1, column 7", i.Cursor(), "cursor 2") // }
i.NextRune() // read "\n", cursor ends up at start of new line // AssertEqual(t, "line 1, column 7", i.Cursor(), "cursor 2")
i.Accept() // i.NextRune() // read "\n", cursor ends up at start of new line
AssertEqual(t, "line 2, column 1", i.Cursor(), "cursor 3") // i.Accept()
for j := 0; j < 10; j++ { // read "with\r\nnewl", cursor end up at "i" // AssertEqual(t, "line 2, column 1", i.Cursor(), "cursor 3")
i.NextRune() // for j := 0; j < 10; j++ { // read "with\r\nnewl", cursor end up at "i"
i.Accept() // i.NextRune()
} // i.Accept()
AssertEqual(t, "line 3, column 5", i.Cursor(), "cursor 4") // }
} // AssertEqual(t, "line 3, column 5", i.Cursor(), "cursor 4")
// }
func TestWhenCallingNextruneAtEndOfFile_EOFIsReturned(t *testing.T) { // TODO FIXME
i := tokenize.NewAPI(strings.NewReader("X")) // func TestWhenCallingNextruneAtEndOfFile_EOFIsReturned(t *testing.T) {
i.NextRune() // i := tokenize.NewAPI(strings.NewReader("X"))
i.Accept() // i.NextRune()
r, err := i.NextRune() // i.Accept()
AssertEqual(t, true, r == utf8.RuneError, "returned rune from NextRune()") // r, err := i.NextRune()
AssertEqual(t, true, err == io.EOF, "returned error from NextRune()") // AssertEqual(t, true, r == utf8.RuneError, "returned rune from NextRune()")
} // AssertEqual(t, true, err == io.EOF, "returned error from NextRune()")
func TestAfterReadingruneAtEndOfFile_EarlierRunesCanStillBeAccessed(t *testing.T) { // }
i := tokenize.NewAPI(strings.NewReader("X")) // TODO FIXME
child := i.Fork() // func TestAfterReadingruneAtEndOfFile_EarlierRunesCanStillBeAccessed(t *testing.T) {
i.NextRune() // i := tokenize.NewAPI(strings.NewReader("X"))
i.Accept() // child := i.Fork()
r, err := i.NextRune() // i.NextRune()
AssertEqual(t, true, r == utf8.RuneError, "returned rune from 2nd NextRune()") // i.Accept()
i.Dispose(child) // brings the read offset back to the start // r, err := i.NextRune()
r, err = i.NextRune() // so here we should see the same rune // AssertEqual(t, true, r == utf8.RuneError, "returned rune from 2nd NextRune()")
AssertEqual(t, 'X', r, "returned rune from 2nd NextRune()") // i.Dispose(child) // brings the read offset back to the start
AssertEqual(t, true, err == nil, "returned error from 2nd NextRune()") // r, err = i.NextRune() // so here we should see the same rune
} // AssertEqual(t, 'X', r, "returned rune from 2nd NextRune()")
// AssertEqual(t, true, err == nil, "returned error from 2nd NextRune()")
// }
func makeTokenizeAPI() *tokenize.API { func makeTokenizeAPI() *tokenize.API {
return tokenize.NewAPI("Testing") return tokenize.NewAPI("Testing")

156
tokenize/tokenizer_whitebox_test.go
View File

@ -4,91 +4,95 @@ import (
"testing" "testing"
) )
func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) { // TODO FIXME
// Create input, accept the first rune. // func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
i := NewAPI("Testing") // // Create input, accept the first rune.
i.NextRune() // i := NewAPI("Testing")
i.Accept() // T // i.NextRune()
AssertEqual(t, "T", i.String(), "accepted rune in input") // i.Accept() // T
// Fork // AssertEqual(t, "T", i.String(), "accepted rune in input")
child := i.Fork() // // Fork
AssertEqual(t, 1, i.stackFrame.offset, "parent offset") // child := i.Fork()
AssertEqual(t, 1, i.stackFrame.offset, "child offset") // AssertEqual(t, 1, i.stackFrame.offset, "parent offset")
// Accept two runes via fork. // AssertEqual(t, 1, i.stackFrame.offset, "child offset")
i.NextRune() // // Accept two runes via fork.
i.Accept() // e // i.NextRune()
i.NextRune() // i.Accept() // e
i.Accept() // s // i.NextRune()
AssertEqual(t, "es", i.String(), "result runes in fork") // i.Accept() // s
AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset") // AssertEqual(t, "es", i.String(), "result runes in fork")
AssertEqual(t, 3, i.stackFrame.offset, "child offset") // AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset")
// Merge fork back into parent // AssertEqual(t, 3, i.stackFrame.offset, "child offset")
i.Merge(child) // // Merge fork back into parent
i.Dispose(child) // i.Merge(child)
AssertEqual(t, "Tes", i.String(), "result runes in parent Input after Merge()") // i.Dispose(child)
AssertEqual(t, 3, i.stackFrame.offset, "parent offset") // AssertEqual(t, "Tes", i.String(), "result runes in parent Input after Merge()")
} // AssertEqual(t, 3, i.stackFrame.offset, "parent offset")
// }
func TestGivenForkedChildWhichAcceptedRune_AfterMerging_RuneEndsUpInParentResult(t *testing.T) { // TODO FIXME
i := NewAPI("Testing") // func TestGivenForkedChildWhichAcceptedRune_AfterMerging_RuneEndsUpInParentResult(t *testing.T) {
i.NextRune() // i := NewAPI("Testing")
i.Accept() // i.NextRune()
f1 := i.Fork() // i.Accept()
i.NextRune() // f1 := i.Fork()
i.Accept() // i.NextRune()
f2 := i.Fork() // i.Accept()
i.NextRune() // f2 := i.Fork()
i.Accept() // i.NextRune()
AssertEqual(t, "s", i.String(), "f2 String()") // i.Accept()
AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A") // AssertEqual(t, "s", i.String(), "f2 String()")
i.Merge(f2) // AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A")
i.Dispose(f2) // i.Merge(f2)
AssertEqual(t, "es", i.String(), "f1 String()") // i.Dispose(f2)
AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") // AssertEqual(t, "es", i.String(), "f1 String()")
i.Merge(f1) // AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
i.Dispose(f1) // i.Merge(f1)
AssertEqual(t, "Tes", i.String(), "top-level API String()") // i.Dispose(f1)
AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") // AssertEqual(t, "Tes", i.String(), "top-level API String()")
} // AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
// }
func TestCallingAcceptAfterNextRune_AcceptsRuneAndMovesReadOffsetForward(t *testing.T) { // TODO FIXME
i := NewAPI("Testing") // func TestCallingAcceptAfterNextRune_AcceptsRuneAndMovesReadOffsetForward(t *testing.T) {
r, _ := i.NextRune() // i := NewAPI("Testing")
AssertEqual(t, 'T', r, "result from 1st call to NextRune()") // r, _ := i.NextRune()
AssertTrue(t, i.lastRune == 'T', "API.lastRune after NextRune() is not 'T'") // AssertEqual(t, 'T', r, "result from 1st call to NextRune()")
AssertTrue(t, i.runeRead, "API.runeRead after NextRune() is not true") // AssertTrue(t, i.lastRune == 'T', "API.lastRune after NextRune() is not 'T'")
i.Accept() // AssertTrue(t, i.runeRead, "API.runeRead after NextRune() is not true")
AssertTrue(t, i.runeRead == false, "API.runeRead after Accept() is not false") // i.Accept()
AssertEqual(t, 1, i.stackFrame.offset, "API.stackFrame.offset") // AssertTrue(t, i.runeRead == false, "API.runeRead after Accept() is not false")
r, _ = i.NextRune() // AssertEqual(t, 1, i.stackFrame.offset, "API.stackFrame.offset")
AssertEqual(t, 'e', r, "result from 2nd call to NextRune()") // r, _ = i.NextRune()
} // AssertEqual(t, 'e', r, "result from 2nd call to NextRune()")
// }
func TestFlushInput(t *testing.T) { // TODO FIXME
api := NewAPI("cool") // func TestFlushInput(t *testing.T) {
// api := NewAPI("cool")
// Flushing without any read data is okay. FlushInput() will return // // Flushing without any read data is okay. FlushInput() will return
// false in this case, and nothing else happens. // // false in this case, and nothing else happens.
AssertTrue(t, api.FlushInput() == false, "flush input at start") // AssertTrue(t, api.FlushInput() == false, "flush input at start")
api.NextRune() // api.NextRune()
api.Accept() // api.Accept()
api.NextRune() // api.NextRune()
api.Accept() // api.Accept()
AssertTrue(t, api.FlushInput() == true, "flush input after reading some data") // AssertTrue(t, api.FlushInput() == true, "flush input after reading some data")
AssertEqual(t, 0, api.stackFrame.offset, "offset after flush input") // AssertEqual(t, 0, api.stackFrame.offset, "offset after flush input")
AssertTrue(t, api.FlushInput() == false, "flush input after flush input") // AssertTrue(t, api.FlushInput() == false, "flush input after flush input")
// Read offset is now zero, but reading should continue after "co". // // Read offset is now zero, but reading should continue after "co".
api.NextRune() // api.NextRune()
api.Accept() // api.Accept()
api.NextRune() // api.NextRune()
api.Accept() // api.Accept()
AssertEqual(t, "cool", api.String(), "end result") // AssertEqual(t, "cool", api.String(), "end result")
} // }
func TestInputFlusherWrapper(t *testing.T) { func TestInputFlusherWrapper(t *testing.T) {
runeA := A.Rune('a') runeA := A.Rune('a')