From e659380a5f935e6145ea901aa1310493223cf9ee Mon Sep 17 00:00:00 2001 From: Maurice Makaay Date: Wed, 17 Jul 2019 23:51:37 +0000 Subject: [PATCH] Implemented an efficient M.DropUntilEndOfLine handler, which is now used in the TOML parser for a dramatic speed increase on comment parsing. --- tokenize/handlers_builtin.go | 69 ++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/tokenize/handlers_builtin.go b/tokenize/handlers_builtin.go index a181ecc..3a7ad41 100644 --- a/tokenize/handlers_builtin.go +++ b/tokenize/handlers_builtin.go @@ -266,25 +266,27 @@ var A = struct { // // Doing so saves you a lot of typing, and it makes your code a lot cleaner. var M = struct { - Drop func(Handler) Handler - Trim func(handler Handler, cutset string) Handler - TrimLeft func(handler Handler, cutset string) Handler - TrimRight func(handler Handler, cutset string) Handler - TrimSpace func(handler Handler) Handler - ToLower func(Handler) Handler - ToUpper func(Handler) Handler - Replace func(handler Handler, replaceWith string) Handler - ByCallback func(Handler, func(string) string) Handler + Drop func(Handler) Handler + DropUntilEndOfLine Handler + Trim func(handler Handler, cutset string) Handler + TrimLeft func(handler Handler, cutset string) Handler + TrimRight func(handler Handler, cutset string) Handler + TrimSpace func(handler Handler) Handler + ToLower func(Handler) Handler + ToUpper func(Handler) Handler + Replace func(handler Handler, replaceWith string) Handler + ByCallback func(Handler, func(string) string) Handler }{ - Drop: ModifyDrop, - Trim: ModifyTrim, - TrimLeft: ModifyTrimLeft, - TrimRight: ModifyTrimRight, - TrimSpace: ModifyTrimSpace, - ToLower: ModifyToLower, - ToUpper: ModifyToUpper, - Replace: ModifyReplace, - ByCallback: ModifyByCallback, + Drop: ModifyDrop, + DropUntilEndOfLine: ModifyDropUntilEndOfLine(), + Trim: ModifyTrim, + TrimLeft: ModifyTrimLeft, + TrimRight: ModifyTrimRight, + TrimSpace: ModifyTrimSpace, + ToLower: ModifyToLower, + ToUpper: ModifyToUpper, + Replace: ModifyReplace, + ByCallback: ModifyByCallback, } // T provides convenient access to a range of Token producers (which in their @@ -698,7 +700,14 @@ func MatchStrNoCase(expected string) Handler { // no output is generated but still a successful match is reported (but the // result will be empty). func MatchOptional(handler Handler) Handler { - return matchMinMax(0, 1, handler, "MatchOptional") + return func(t *API) bool { + child := t.Fork() + if handler(t) { + t.Merge(child) + } + t.Dispose(child) + return true + } } // MatchSeq creates a Handler that checks if the provided Handlers can be @@ -1542,6 +1551,28 @@ func ModifyDrop(handler Handler) Handler { } } +// ModifyDropUntilEndOfLine creates a Handler that drops all input until an end of line +// (or end of file). This handler is typically used when ignoring any input data after +// a comment start like '#' or '//' when parsing code or configuration data. +func ModifyDropUntilEndOfLine() Handler { + return func(t *API) bool { + for { + b, err := t.PeekByte(0) + if err != nil { + if err == io.EOF { + return true + } else { + return false + } + } + if b == '\n' { + return true + } + t.skipBytes(b) + } + } +} + // ModifyTrim creates a Handler that checks if the provided Handler applies. // If it does, then its output is taken and characters from the provided // cutset are trimmed from both the left and the right of the output.