go-parsekit/tokenhandler.go

package parsekit

import (
	"fmt"
	"runtime"
)

// TokenHandler is the function type that is involved in turning a low level
// stream of UTF8 runes into lexical tokens. Its purpose is to check if input
// data matches some kind of pattern and to report back the token(s).
//
// A TokenHandler function gets a TokenAPI as its input and returns a boolean to
// indicate whether or not it found a match on the input. The TokenAPI is used
// for retrieving input data to match against and for reporting back results.
type TokenHandler func(t *TokenAPI) bool

// NextRune retrieves the next rune from the input.
//
// It returns the rune and a boolean. The boolean will be false in case an
// invalid UTF8 rune or the end of the file was encountered.
//
// After retrieving a rune, Accept() or Skip() can be called to respectively add
// the rune to the TokenAPIold's string buffer or to fully ignore it. This way,
// a TokenHandler has full control over what runes are significant for the
// resulting output of that TokenHandler.
//
// After using NextRune(), this method can not be reinvoked, until the last read
// rune is explicitly accepted or skipped as described above.
// func (t *TokenAPIold) NextRune() (rune, bool) {
// 	if t.lastRune != nil {
// 		caller, filepos := getCaller(1)
// 		panic(fmt.Sprintf(
// 			"TokenHandler bug: NextRune() was called from %s at %s "+
// 				"without accepting or skipping the previously read rune", caller, filepos))
// 	}
// 	r, w, ok := 'X', 10, true // t.input.peek(t.inputOffset)
// 	t.lastRune = &runeInfo{r, w, ok}
// 	if ok {
// 		t.result.Input = append(t.result.Input, r)
// 	}
// 	return r, ok
// }

// Fork splits off a child TokenAPIold, containing the same input cursor position
//  as the parent TokenAPIold, but with all other data in a fresh state.
//
// By forking, a TokenHandler function can freely work with a TokenAPIold, without
// affecting the parent TokenAPIold. This is for example useful when the
// TokenHandler function must perform some form of lookahead.
//
// When a successful match was found, the TokenHandler function can call
// TokenAPIold.Merge() on the forked child to have the resulting output added
// to the parent TokenAPIold.
//
// When no match was found, the forked child can simply be discarded.
//
// Example case: A TokenHandler checks for a sequence of runes: 'a', 'b', 'c', 'd'.
// This is done in 4 steps and only after finishing all steps, the TokenHandler
// function can confirm a successful match. The TokenHandler function for this
// case could look like this (yes, it's naive, but it shows the point):
// TODO make proper tested example
//
//     func MatchAbcd(t *TokenAPIold) bool {
//         child := t.Fork() // fork to keep m from input untouched
//         for _, letter := []rune {'a', 'b', 'c', 'd'} {
//             if r, ok := t.NextRune(); !ok || r != letter {
//                 return false // report mismatch, t is left untouched
//             }
//             child.Accept() // add rune to child output
//         }
//         child.Merge() // we have a match, add resulting output to parent
//         return true // and report the successful match
//     }

// Accept will add the last rune as read by TokenAPIold.NextRune() to the resulting
// output of the TokenAPIold.
// func (t *TokenAPIold) Accept() {
// 	t.checkAllowedCall("Accept()")
// 	t.buffer = append(t.buffer, t.lastRune.Rune)
// 	t.result.Accepted = append(t.result.Accepted, t.lastRune.Rune)
// 	t.inputOffset += t.lastRune.ByteSize
// 	t.lastRune = nil
// }

// Skip will ignore the last rune as read by NextRune().
// func (t *TokenAPIold) Skip() {
// 	t.checkAllowedCall("Skip()")
// 	t.inputOffset += t.lastRune.ByteSize
// 	t.lastRune = nil
// }

// func (t *TokenAPIold) checkAllowedCall(name string) {
// 	if t.lastRune == nil {
// 		caller, filepos := getCaller(2)
// 		panic(fmt.Sprintf(
// 			"TokenHandler bug: %s was called from %s at %s without a prior call to NextRune()",
// 			name, caller, filepos))
// 	}
// 	if !t.lastRune.OK {
// 		caller, filepos := getCaller(2)
// 		panic(fmt.Sprintf(
// 			"TokenHandler bug: %s was called from %s at %s, but prior call to NextRune() "+
// 				"did not return OK (EOF or invalid rune)", name, caller, filepos))
// 	}
// }

// AddToken is used to add a token to the results of the TokenHandler.
// func (t *TokenAPIold) AddToken(tok *Token) {
// 	t.result.Tokens = append(t.result.Tokens, tok)
// }

// Merge merges the resulting output from a forked child TokenAPIold back into
// its parent: The runes that are accepted in the child are added to the parent
// runes and the parent's input cursor position is advanced to the child's
// cursor position.
//
// After the merge, the child TokenAPIold is reset so it can immediately be
// reused for performing another match (all data are cleared, except for the
// input offset which is kept at its current position).
// func (t *TokenAPIold) Merge() bool {
// 	if t.parent == nil {
// 		panic("TokenHandler bug: Cannot call Merge a a non-forked MatchDialog")
// 	}
// 	t.parent.buffer = append(t.parent.buffer, t.result.Accepted...)
// 	t.parent.result.Input = append(t.parent.result.Input, t.result.Input...)
// 	t.parent.result.Accepted = append(t.parent.result.Accepted, t.result.Accepted...)
// 	t.parent.result.Tokens = append(t.parent.result.Tokens, t.result.Tokens...)
// 	t.parent.inputOffset = t.inputOffset
// 	t.result = &TokResult{}
// 	return true
// }

func getCaller(depth int) (string, string) {
	// No error handling, because we call this method ourselves with safe depth values.
	pc, file, line, _ := runtime.Caller(depth + 1)
	filepos := fmt.Sprintf("%s:%d", file, line)
	caller := runtime.FuncForPC(pc)
	return caller.Name(), filepos
}