diff --git a/tokenize/handlers_builtin.go b/tokenize/handlers_builtin.go index e73acd8..095965e 100644 --- a/tokenize/handlers_builtin.go +++ b/tokenize/handlers_builtin.go @@ -355,7 +355,7 @@ func MatchByte(expected byte) Handler { // MatchRune creates a Handler function that matches against the provided rune. func MatchRune(expected rune) Handler { - if expected <= 255 { + if expected <= 127 { return MatchByte(byte(expected)) } return func(t *API) bool { @@ -448,7 +448,7 @@ func MatchRuneRange(start rune, end rune) Handler { if end < start { callerPanic("MatchRuneRange", "Handler: {name} definition error at {caller}: start %q must not be < end %q", start, end) } - if end <= 255 { + if end <= 127 { return MatchByteRange(byte(start), byte(end)) } return func(t *API) bool { @@ -621,7 +621,7 @@ func MatchStr(expected string) Handler { return func(t *API) bool { offset := 0 for _, e := range expectedRunes { - if e <= 255 { + if e <= 127 { b, err := t.PeekByte(offset) if err != nil || b != byte(e) { return false @@ -643,15 +643,16 @@ func MatchStr(expected string) Handler { // MatchStrNoCase creates a Handler that matches the input against the // provided string in a case-insensitive manner. func MatchStrNoCase(expected string) Handler { - l := len([]rune(expected)) + l := utf8.RuneCountInString(expected) return func(t *API) bool { matches := make([]rune, l) width := 0 - for i, e := range expected { - if e <= 255 { + i := 0 + for _, e := range expected { + if e <= 127 { b, err := t.PeekByte(width) - if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(rune(e))) { + if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(e)) { return false } matches[i] = rune(b) @@ -664,6 +665,7 @@ func MatchStrNoCase(expected string) Handler { matches[i] = r width += w } + i++ } t.acceptRunes(width, matches...) return true diff --git a/tokenize/handlers_builtin_test.go b/tokenize/handlers_builtin_test.go index e436461..1d754e2 100644 --- a/tokenize/handlers_builtin_test.go +++ b/tokenize/handlers_builtin_test.go @@ -116,8 +116,10 @@ func TestAtoms(t *testing.T) { {"dd", a.RuneRange('b', 'e'), true, "d"}, {"ee", a.RuneRange('b', 'e'), true, "e"}, {"ff", a.RuneRange('b', 'e'), false, ""}, - {"Hello, world!", a.Str("Hello"), true, "Hello"}, - {"HellÖ, world!", a.StrNoCase("hellö"), true, "HellÖ"}, + {"Hello, world 1!", a.Str("Hello"), true, "Hello"}, + {"Hello, world 2!", a.StrNoCase("hElLo"), true, "Hello"}, + {"H♥llÖ, wÖrld 3!", a.Str("H♥llÖ"), true, "H♥llÖ"}, + {"H♥llÖ, world 4!", a.StrNoCase("h♥llö"), true, "H♥llÖ"}, {"+X", a.Runes('+', '-', '*', '/'), true, "+"}, {"-X", a.Runes('+', '-', '*', '/'), true, "-"}, {"*X", a.Runes('+', '-', '*', '/'), true, "*"},