OpenSourceProjects
/
go-parsekit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Small bugfix for the rune-to-byte-fallback code and added byte-support to the Str and StrNoCase matchers.

This commit is contained in:
Maurice Makaay 2019-07-16 07:35:06 +00:00
parent 4cfdbafa6e
commit 06faabdfe2
2 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tokenize/handlers_builtin.go
View File

@ -355,7 +355,7 @@ func MatchByte(expected byte) Handler {
// MatchRune creates a Handler function that matches against the provided rune. // MatchRune creates a Handler function that matches against the provided rune.
func MatchRune(expected rune) Handler { func MatchRune(expected rune) Handler {
if expected <= 255 { if expected <= 127 {
return MatchByte(byte(expected)) return MatchByte(byte(expected))
} }
return func(t *API) bool { return func(t *API) bool {
@ -448,7 +448,7 @@ func MatchRuneRange(start rune, end rune) Handler {
if end < start { if end < start {
callerPanic("MatchRuneRange", "Handler: {name} definition error at {caller}: start %q must not be < end %q", start, end) callerPanic("MatchRuneRange", "Handler: {name} definition error at {caller}: start %q must not be < end %q", start, end)
} }
if end <= 255 { if end <= 127 {
return MatchByteRange(byte(start), byte(end)) return MatchByteRange(byte(start), byte(end))
} }
return func(t *API) bool { return func(t *API) bool {
@ -621,7 +621,7 @@ func MatchStr(expected string) Handler {
return func(t *API) bool { return func(t *API) bool {
offset := 0 offset := 0
for _, e := range expectedRunes { for _, e := range expectedRunes {
if e <= 255 { if e <= 127 {
b, err := t.PeekByte(offset) b, err := t.PeekByte(offset)
if err != nil || b != byte(e) { if err != nil || b != byte(e) {
return false return false
@ -643,15 +643,16 @@ func MatchStr(expected string) Handler {
// MatchStrNoCase creates a Handler that matches the input against the // MatchStrNoCase creates a Handler that matches the input against the
// provided string in a case-insensitive manner. // provided string in a case-insensitive manner.
func MatchStrNoCase(expected string) Handler { func MatchStrNoCase(expected string) Handler {
l := len([]rune(expected)) l := utf8.RuneCountInString(expected)
return func(t *API) bool { return func(t *API) bool {
matches := make([]rune, l) matches := make([]rune, l)
width := 0 width := 0
for i, e := range expected { i := 0
if e <= 255 { for _, e := range expected {
if e <= 127 {
b, err := t.PeekByte(width) b, err := t.PeekByte(width)
if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(rune(e))) { if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(e)) {
return false return false
} }
matches[i] = rune(b) matches[i] = rune(b)
@ -664,6 +665,7 @@ func MatchStrNoCase(expected string) Handler {
matches[i] = r matches[i] = r
width += w width += w
} }
i++
} }
t.acceptRunes(width, matches...) t.acceptRunes(width, matches...)
return true return true

6
tokenize/handlers_builtin_test.go
View File

@ -116,8 +116,10 @@ func TestAtoms(t *testing.T) {
{"dd", a.RuneRange('b', 'e'), true, "d"}, {"dd", a.RuneRange('b', 'e'), true, "d"},
{"ee", a.RuneRange('b', 'e'), true, "e"}, {"ee", a.RuneRange('b', 'e'), true, "e"},
{"ff", a.RuneRange('b', 'e'), false, ""}, {"ff", a.RuneRange('b', 'e'), false, ""},
{"Hello, world!", a.Str("Hello"), true, "Hello"}, {"Hello, world 1!", a.Str("Hello"), true, "Hello"},
{"HellÖ, world!", a.StrNoCase("hellö"), true, "HellÖ"}, {"Hello, world 2!", a.StrNoCase("hElLo"), true, "Hello"},
{"H♥llÖ, wÖrld 3!", a.Str("H♥llÖ"), true, "H♥llÖ"},
{"H♥llÖ, world 4!", a.StrNoCase("h♥llö"), true, "H♥llÖ"},
{"+X", a.Runes('+', '-', '*', '/'), true, "+"}, {"+X", a.Runes('+', '-', '*', '/'), true, "+"},
{"-X", a.Runes('+', '-', '*', '/'), true, "-"}, {"-X", a.Runes('+', '-', '*', '/'), true, "-"},
{"*X", a.Runes('+', '-', '*', '/'), true, "*"}, {"*X", a.Runes('+', '-', '*', '/'), true, "*"},