From e659380a5f935e6145ea901aa1310493223cf9ee Mon Sep 17 00:00:00 2001
From: Maurice Makaay <maurice@makaay.nl>
Date: Wed, 17 Jul 2019 23:51:37 +0000
Subject: [PATCH] Implemented an efficient M.DropUntilEndOfLine handler, which
 is now used in the TOML parser for a dramatic speed increase on comment
 parsing.

---
 tokenize/handlers_builtin.go | 69 ++++++++++++++++++++++++++----------
 1 file changed, 50 insertions(+), 19 deletions(-)

diff --git a/tokenize/handlers_builtin.go b/tokenize/handlers_builtin.go
index a181ecc..3a7ad41 100644
--- a/tokenize/handlers_builtin.go
+++ b/tokenize/handlers_builtin.go
@@ -266,25 +266,27 @@ var A = struct {
 //
 // Doing so saves you a lot of typing, and it makes your code a lot cleaner.
 var M = struct {
-	Drop       func(Handler) Handler
-	Trim       func(handler Handler, cutset string) Handler
-	TrimLeft   func(handler Handler, cutset string) Handler
-	TrimRight  func(handler Handler, cutset string) Handler
-	TrimSpace  func(handler Handler) Handler
-	ToLower    func(Handler) Handler
-	ToUpper    func(Handler) Handler
-	Replace    func(handler Handler, replaceWith string) Handler
-	ByCallback func(Handler, func(string) string) Handler
+	Drop               func(Handler) Handler
+	DropUntilEndOfLine Handler
+	Trim               func(handler Handler, cutset string) Handler
+	TrimLeft           func(handler Handler, cutset string) Handler
+	TrimRight          func(handler Handler, cutset string) Handler
+	TrimSpace          func(handler Handler) Handler
+	ToLower            func(Handler) Handler
+	ToUpper            func(Handler) Handler
+	Replace            func(handler Handler, replaceWith string) Handler
+	ByCallback         func(Handler, func(string) string) Handler
 }{
-	Drop:       ModifyDrop,
-	Trim:       ModifyTrim,
-	TrimLeft:   ModifyTrimLeft,
-	TrimRight:  ModifyTrimRight,
-	TrimSpace:  ModifyTrimSpace,
-	ToLower:    ModifyToLower,
-	ToUpper:    ModifyToUpper,
-	Replace:    ModifyReplace,
-	ByCallback: ModifyByCallback,
+	Drop:               ModifyDrop,
+	DropUntilEndOfLine: ModifyDropUntilEndOfLine(),
+	Trim:               ModifyTrim,
+	TrimLeft:           ModifyTrimLeft,
+	TrimRight:          ModifyTrimRight,
+	TrimSpace:          ModifyTrimSpace,
+	ToLower:            ModifyToLower,
+	ToUpper:            ModifyToUpper,
+	Replace:            ModifyReplace,
+	ByCallback:         ModifyByCallback,
 }
 
 // T provides convenient access to a range of Token producers (which in their
@@ -698,7 +700,14 @@ func MatchStrNoCase(expected string) Handler {
 // no output is generated but still a successful match is reported (but the
 // result will be empty).
 func MatchOptional(handler Handler) Handler {
-	return matchMinMax(0, 1, handler, "MatchOptional")
+	return func(t *API) bool {
+		child := t.Fork()
+		if handler(t) {
+			t.Merge(child)
+		}
+		t.Dispose(child)
+		return true
+	}
 }
 
 // MatchSeq creates a Handler that checks if the provided Handlers can be
@@ -1542,6 +1551,28 @@ func ModifyDrop(handler Handler) Handler {
 	}
 }
 
+// ModifyDropUntilEndOfLine creates a Handler that drops all input until an end of line
+// (or end of file). This handler is typically used when ignoring any input data after
+// a comment start like '#' or '//' when parsing code or configuration data.
+func ModifyDropUntilEndOfLine() Handler {
+	return func(t *API) bool {
+		for {
+			b, err := t.PeekByte(0)
+			if err != nil {
+				if err == io.EOF {
+					return true
+				} else {
+					return false
+				}
+			}
+			if b == '\n' {
+				return true
+			}
+			t.skipBytes(b)
+		}
+	}
+}
+
 // ModifyTrim creates a Handler that checks if the provided Handler applies.
 // If it does, then its output is taken and characters from the provided
 // cutset are trimmed from both the left and the right of the output.