OpenSourceProjects
/
go-parsekit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Small bugfix for the rune-to-byte-fallback code and added byte-support to the Str and StrNoCase matchers.

This commit is contained in:
Maurice Makaay 2019-07-16 07:35:06 +00:00
parent 4cfdbafa6e
commit 06faabdfe2
2 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tokenize/handlers_builtin.go
View File

@ -355,7 +355,7 @@ func MatchByte(expected byte) Handler {
// MatchRune creates a Handler function that matches against the provided rune.
func MatchRune(expected rune) Handler {
if expected <= 255 {
if expected <= 127 {
return MatchByte(byte(expected))
}
return func(t *API) bool {
@ -448,7 +448,7 @@ func MatchRuneRange(start rune, end rune) Handler {
if end < start {
callerPanic("MatchRuneRange", "Handler: {name} definition error at {caller}: start %q must not be < end %q", start, end)
}
if end <= 255 {
if end <= 127 {
return MatchByteRange(byte(start), byte(end))
}
return func(t *API) bool {
@ -621,7 +621,7 @@ func MatchStr(expected string) Handler {
return func(t *API) bool {
offset := 0
for _, e := range expectedRunes {
if e <= 255 {
if e <= 127 {
b, err := t.PeekByte(offset)
if err != nil || b != byte(e) {
return false
@ -643,15 +643,16 @@ func MatchStr(expected string) Handler {
// MatchStrNoCase creates a Handler that matches the input against the
// provided string in a case-insensitive manner.
func MatchStrNoCase(expected string) Handler {
l := len([]rune(expected))
l := utf8.RuneCountInString(expected)
return func(t *API) bool {
matches := make([]rune, l)
width := 0
for i, e := range expected {
if e <= 255 {
i := 0
for _, e := range expected {
if e <= 127 {
b, err := t.PeekByte(width)
if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(rune(e))) {
if err != nil || (b != byte(e) && unicode.ToUpper(rune(b)) != unicode.ToUpper(e)) {
return false
}
matches[i] = rune(b)
@ -664,6 +665,7 @@ func MatchStrNoCase(expected string) Handler {
matches[i] = r
width += w
}
i++
}
t.acceptRunes(width, matches...)
return true

6
tokenize/handlers_builtin_test.go
View File

@ -116,8 +116,10 @@ func TestAtoms(t *testing.T) {
{"dd", a.RuneRange('b', 'e'), true, "d"},
{"ee", a.RuneRange('b', 'e'), true, "e"},
{"ff", a.RuneRange('b', 'e'), false, ""},
{"Hello, world!", a.Str("Hello"), true, "Hello"},
{"HellÖ, world!", a.StrNoCase("hellö"), true, "HellÖ"},
{"Hello, world 1!", a.Str("Hello"), true, "Hello"},
{"Hello, world 2!", a.StrNoCase("hElLo"), true, "Hello"},
{"H♥llÖ, wÖrld 3!", a.Str("H♥llÖ"), true, "H♥llÖ"},
{"H♥llÖ, world 4!", a.StrNoCase("h♥llö"), true, "H♥llÖ"},
{"+X", a.Runes('+', '-', '*', '/'), true, "+"},
{"-X", a.Runes('+', '-', '*', '/'), true, "-"},
{"*X", a.Runes('+', '-', '*', '/'), true, "*"},