OpenSourceProjects
/
go-parsekit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Further code cleaning for the interaction between ParseAPI and TokenAPI. Extra atoms added, also one based on a callback which can accept single runes based on thhat callback function.

This commit is contained in:
Maurice Makaay 2019-06-07 15:48:49 +00:00
parent 98d2db0374
commit 9a5bf8b9af
11 changed files with 286 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
examples/example_basiccalculator1_test.go
View File

@ -3,8 +3,8 @@
// //
// 10 + 20 - 8+4 // 10 + 20 - 8+4
// //
// So positive numbers that can be either added or substracted, and whitespace // So positive numbers that can be either added or substracted, and blanks
// is ignored. // around numbers are ignored.
package examples package examples
import ( import (
@ -69,9 +69,9 @@ type simpleCalculator struct {
op int64 // represents operation for next term (+1 = add, -1 = subtract) op int64 // represents operation for next term (+1 = add, -1 = subtract)
} }
// A definition of an int64, which conveniently drops surrounding whitespace. // A definition of an int64, which conveniently drops surrounding blanks.
var dropWhitespace = parsekit.M.Drop(parsekit.C.Opt(parsekit.A.Whitespace)) var dropBlank = parsekit.M.Drop(parsekit.C.Opt(parsekit.A.Blank))
var bareInteger = parsekit.C.Seq(dropWhitespace, parsekit.A.Integer, dropWhitespace) var bareInteger = parsekit.C.Seq(dropBlank, parsekit.A.Integer, dropBlank)
var int64Token = parsekit.T.Int64(nil, bareInteger) var int64Token = parsekit.T.Int64(nil, bareInteger)
func (c *simpleCalculator) number(p *parsekit.ParseAPI) { func (c *simpleCalculator) number(p *parsekit.ParseAPI) {

4
examples/example_basiccalculator2_test.go
View File

@ -130,7 +130,7 @@ func (c *calculator) term(p *parsekit.ParseAPI) {
// <factor> = <space> (FLOAT | LPAREN <expr> RPAREN) <space> // <factor> = <space> (FLOAT | LPAREN <expr> RPAREN) <space>
func (c *calculator) factor(p *parsekit.ParseAPI) { func (c *calculator) factor(p *parsekit.ParseAPI) {
var A, T = parsekit.A, parsekit.T var A, T = parsekit.A, parsekit.T
p.On(A.Whitespace).Skip() p.On(A.Blank).Skip()
switch { switch {
case p.On(T.Float64(nil, A.Signed(A.Float))).Accept(): case p.On(T.Float64(nil, A.Signed(A.Float))).Accept():
value := p.Result().Value(0).(float64) value := p.Result().Value(0).(float64)
@ -147,7 +147,7 @@ func (c *calculator) factor(p *parsekit.ParseAPI) {
p.UnexpectedInput("factor or (expression)") p.UnexpectedInput("factor or (expression)")
return return
} }
p.On(A.Whitespace).Skip() p.On(A.Blank).Skip()
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

2
examples/example_dutchpostcode_test.go
View File

@ -65,7 +65,7 @@ func createPostcodeTokenizer() *parsekit.Tokenizer {
pcDigits := C.Seq(digitNotZero, C.Rep(3, A.Digit)) pcDigits := C.Seq(digitNotZero, C.Rep(3, A.Digit))
pcLetter := C.Any(A.ASCIILower, A.ASCIIUpper) pcLetter := C.Any(A.ASCIILower, A.ASCIIUpper)
pcLetters := M.ToUpper(C.Seq(pcLetter, pcLetter)) pcLetters := M.ToUpper(C.Seq(pcLetter, pcLetter))
space := M.Replace(C.Opt(A.Whitespace), " ") space := M.Replace(C.Opt(A.Blank), " ")
postcode := C.Seq(T.Str("PCD", pcDigits), space, T.Str("PCL", pcLetters), A.EndOfFile) postcode := C.Seq(T.Str("PCD", pcDigits), space, T.Str("PCL", pcLetters), A.EndOfFile)
// Create a Tokenizer that wraps the 'postcode' TokenHandler and allows // Create a Tokenizer that wraps the 'postcode' TokenHandler and allows

4
examples/example_helloManyStateParser_test.go
View File

@ -90,7 +90,7 @@ func (h *helloparser1) start(p *parsekit.ParseAPI) {
func (h *helloparser1) comma(p *parsekit.ParseAPI) { func (h *helloparser1) comma(p *parsekit.ParseAPI) {
a := parsekit.A a := parsekit.A
switch { switch {
case p.On(a.Whitespace).Skip(): case p.On(a.Blank).Skip():
p.Handle(h.comma) p.Handle(h.comma)
case p.On(a.Comma).Skip(): case p.On(a.Comma).Skip():
p.Handle(h.startName) p.Handle(h.startName)
@ -102,7 +102,7 @@ func (h *helloparser1) comma(p *parsekit.ParseAPI) {
func (h *helloparser1) startName(p *parsekit.ParseAPI) { func (h *helloparser1) startName(p *parsekit.ParseAPI) {
c, a := parsekit.C, parsekit.A c, a := parsekit.C, parsekit.A
switch { switch {
case p.On(a.Whitespace).Skip(): case p.On(a.Blank).Skip():
p.Handle(h.startName) p.Handle(h.startName)
case p.On(c.Not(a.Excl)).Stay(): case p.On(c.Not(a.Excl)).Stay():
p.Handle(h.name) p.Handle(h.name)

4
examples/example_helloParserCombinator_test.go
View File

@ -54,8 +54,8 @@ func createHelloTokenizer() *parsekit.Tokenizer {
// that does all the work. The 'greeting' TokenHandler matches the whole input and // that does all the work. The 'greeting' TokenHandler matches the whole input and
// drops all but the name from it. // drops all but the name from it.
hello := a.StrNoCase("hello") hello := a.StrNoCase("hello")
comma := c.Seq(c.Opt(a.Whitespace), a.Comma, c.Opt(a.Whitespace)) comma := c.Seq(c.Opt(a.Blank), a.Comma, c.Opt(a.Blank))
separator := c.Any(comma, a.Whitespace) separator := c.Any(comma, a.Blank)
name := c.OneOrMore(c.Not(a.Excl)) name := c.OneOrMore(c.Not(a.Excl))
greeting := c.Seq(m.Drop(hello), m.Drop(separator), name, m.Drop(a.Excl), a.EndOfFile) greeting := c.Seq(m.Drop(hello), m.Drop(separator), name, m.Drop(a.Excl), a.EndOfFile)

2
examples/example_helloSingleStateParser_test.go
View File

@ -84,7 +84,7 @@ func (h *helloparser2) start(p *parsekit.ParseAPI) {
p.Error("the greeting is not being friendly") p.Error("the greeting is not being friendly")
return return
} }
if !p.On(c.Seq(c.Opt(a.Whitespace), a.Comma, c.Opt(a.Whitespace))).Skip() { if !p.On(c.Seq(c.Opt(a.Blank), a.Comma, c.Opt(a.Blank))).Skip() {
p.Error("the greeting is not properly separated") p.Error("the greeting is not properly separated")
return return
} }

45
parseapi.go
View File

@ -66,7 +66,7 @@ func (p *ParseAPI) checkForLoops() {
// //
// So an example chain could look like this: // So an example chain could look like this:
// //
// p.On(parsekit.A.Whitespace).Skip() // p.On(parsekit.A.Blank).Skip()
// //
// The chain as a whole returns a boolean that indicates whether or not at match // The chain as a whole returns a boolean that indicates whether or not at match
// was found. When no match was found, false is returned and Skip() and Accept() // was found. When no match was found, false is returned and Skip() and Accept()
@ -103,31 +103,36 @@ func (p *ParseAPI) On(tokenHandler TokenHandler) *ParseAPIOnAction {
ok := tokenHandler(child) ok := tokenHandler(child)
return &ParseAPIOnAction{ return &ParseAPIOnAction{
parseAPI: p, parseAPI: p,
tokenAPI: child, tokenAPI: p.tokenAPI,
ok: ok, forkedTokenAPI: child,
ok: ok,
} }
} }
// ParseAPIOnAction is a struct that is used for building the On()-method chain. // ParseAPIOnAction is a struct that is used for building the On()-method chain.
// The On() method will return an initialized struct of this type. // The On() method will return an initialized struct of this type.
type ParseAPIOnAction struct { type ParseAPIOnAction struct {
parseAPI *ParseAPI parseAPI *ParseAPI
tokenAPI *TokenAPI tokenAPI *TokenAPI
ok bool forkedTokenAPI *TokenAPI
ok bool
} }
// Accept tells the parser to move the read cursor past a match that was // Accept tells the parser to move the read cursor past a match that was
// found, and to make the TokenHandlerResult from the TokenAPI available in the // found by a TokenHandler, and to make the TokenHandlerResult from the
// ParseAPI through the ParseAPI.Result() method. // TokenAPI available in the ParseAPI through the ParseAPI.Result() method.
// //
// Returns true in case a match was found. // Returns true in case a match was found.
// When no match was found, then no action is taken and false is returned. // When no match was found, then no action is taken and false is returned.
func (a *ParseAPIOnAction) Accept() bool { func (a *ParseAPIOnAction) Accept() bool {
if a.ok { if a.ok {
a.tokenAPI.Merge() a.forkedTokenAPI.Merge()
a.flushReader() a.parseAPI.result = a.tokenAPI.Result()
a.parseAPI.result = a.tokenAPI.root.result a.tokenAPI.detachChilds()
if a.tokenAPI.flushReader() {
a.parseAPI.initLoopCheck()
}
} }
return a.ok return a.ok
} }
@ -145,10 +150,12 @@ func (a *ParseAPIOnAction) Accept() bool {
func (a *ParseAPIOnAction) Skip() bool { func (a *ParseAPIOnAction) Skip() bool {
if a.ok { if a.ok {
a.parseAPI.result = nil a.parseAPI.result = nil
a.tokenAPI.clearResults() a.forkedTokenAPI.clearResults()
a.tokenAPI.syncCursorTo(a.tokenAPI.root)
a.tokenAPI.detachChilds() a.tokenAPI.detachChilds()
a.flushReader() a.forkedTokenAPI.syncCursorTo(a.tokenAPI)
if a.tokenAPI.flushReader() {
a.parseAPI.initLoopCheck()
}
} }
return a.ok return a.ok
} }
@ -170,14 +177,6 @@ func (a *ParseAPIOnAction) Stay() bool {
return a.ok return a.ok
} }
func (a *ParseAPIOnAction) flushReader() {
if a.tokenAPI.result.offset > 0 {
a.tokenAPI.root.reader.Flush(a.tokenAPI.root.result.offset)
a.tokenAPI.root.result.offset = 0
a.parseAPI.initLoopCheck()
}
}
// Result returns a TokenHandlerResult struct, containing results as produced by the // Result returns a TokenHandlerResult struct, containing results as produced by the
// last ParseAPI.On().Accept() call. // last ParseAPI.On().Accept() call.
func (p *ParseAPI) Result() *TokenHandlerResult { func (p *ParseAPI) Result() *TokenHandlerResult {

2
parser_test.go
View File

@ -298,7 +298,7 @@ func TestGivenLoopingParserDefinition_ParserPanics(t *testing.T) {
// p.On(c.Max(5, a.AnyRune)) // p.On(c.Max(5, a.AnyRune))
// //
// The problem here is that Max(5, ...) will also match when there is // The problem here is that Max(5, ...) will also match when there is
// no more input, since Max(5, ---) is actually MinMax(0, 5, ...). // no more input, since Max(5, ...) is actually MinMax(0, 5, ...).
// Therefore the loop will never stop. Solving the loop was simple: // Therefore the loop will never stop. Solving the loop was simple:
// //
// p.On(c.MinMax(1, 5, a.AnyRune)) // p.On(c.MinMax(1, 5, a.AnyRune))

18
tokenapi.go
View File

@ -8,15 +8,15 @@ import (
) )
// TokenAPI wraps a parsekit.reader and its purpose is to retrieve data from // TokenAPI wraps a parsekit.reader and its purpose is to retrieve data from
// the reader and to report back tokenizing results. For easy lookahead support, // a parsekit.reader.Reader and to report back tokenizing results. For easy
// a forking strategy is provided. // lookahead support, a forking strategy is provided.
// //
// BASIC OPERATION: // BASIC OPERATION:
// //
// To retrieve the next rune from the TokenAPI, call the NextRune() method. // To retrieve the next rune from the TokenAPI, call the NextRune() method.
// //
// When the rune is to be accepted as input, call the method Accept(). The rune // When the rune is to be accepted as input, call the method Accept(). The rune
// is then added to the results of the TokenAPI and the read cursor is moved // is then added to the result runes of the TokenAPI and the read cursor is moved
// forward. // forward.
// //
// By invoking NextRune() + Accept() multiple times, the result can be extended // By invoking NextRune() + Accept() multiple times, the result can be extended
@ -63,7 +63,6 @@ import (
// no bookkeeping has to be implemented when implementing a parser. // no bookkeeping has to be implemented when implementing a parser.
type TokenAPI struct { type TokenAPI struct {
reader *reader.Reader reader *reader.Reader
root *TokenAPI // the root TokenAPI
parent *TokenAPI // parent TokenAPI in case this TokenAPI is a fork child parent *TokenAPI // parent TokenAPI in case this TokenAPI is a fork child
child *TokenAPI // child TokenAPI in case this TokenAPI is a fork parent child *TokenAPI // child TokenAPI in case this TokenAPI is a fork parent
result *TokenHandlerResult // results as produced by a TokenHandler (runes, Tokens, cursor position) result *TokenHandlerResult // results as produced by a TokenHandler (runes, Tokens, cursor position)
@ -75,7 +74,6 @@ func NewTokenAPI(r io.Reader) *TokenAPI {
reader: reader.New(r), reader: reader.New(r),
result: newTokenHandlerResult(), result: newTokenHandlerResult(),
} }
input.root = input // TODO remove this one from root input, input.root == nil is also a good check for "is root?".
return input return input
} }
@ -141,7 +139,6 @@ func (i *TokenAPI) Fork() *TokenAPI {
// Create the new fork. // Create the new fork.
child := &TokenAPI{ child := &TokenAPI{
reader: i.reader, reader: i.reader,
root: i.root,
parent: i, parent: i,
} }
child.result = newTokenHandlerResult() child.result = newTokenHandlerResult()
@ -200,6 +197,15 @@ func (i *TokenAPI) detachChildsRecurse() {
i.parent = nil i.parent = nil
} }
func (i *TokenAPI) flushReader() bool {
if i.result.offset > 0 {
i.reader.Flush(i.result.offset)
i.result.offset = 0
return true
}
return false
}
// Result returns the TokenHandlerResult data for the TokenAPI. The returned struct // Result returns the TokenHandlerResult data for the TokenAPI. The returned struct
// can be used to retrieve and to modify result data. // can be used to retrieve and to modify result data.
func (i *TokenAPI) Result() *TokenHandlerResult { func (i *TokenAPI) Result() *TokenHandlerResult {

393
tokenhandlers_builtin.go
View File

@ -62,160 +62,168 @@ var C = struct {
// //
// Doing so saves you a lot of typing, and it makes your code a lot cleaner. // Doing so saves you a lot of typing, and it makes your code a lot cleaner.
var A = struct { var A = struct {
Rune func(rune) TokenHandler Rune func(rune) TokenHandler
Runes func(...rune) TokenHandler Runes func(...rune) TokenHandler
RuneRange func(rune, rune) TokenHandler RuneRange func(rune, rune) TokenHandler
Str func(string) TokenHandler Str func(string) TokenHandler
StrNoCase func(string) TokenHandler StrNoCase func(string) TokenHandler
EndOfFile TokenHandler EndOfFile TokenHandler
AnyRune TokenHandler AnyRune TokenHandler
ValidRune TokenHandler ValidRune TokenHandler
Space TokenHandler Space TokenHandler
Tab TokenHandler Tab TokenHandler
CR TokenHandler CR TokenHandler
LF TokenHandler LF TokenHandler
CRLF TokenHandler CRLF TokenHandler
Excl TokenHandler Excl TokenHandler
DoubleQuote TokenHandler DoubleQuote TokenHandler
Hash TokenHandler Hash TokenHandler
Dollar TokenHandler Dollar TokenHandler
Percent TokenHandler Percent TokenHandler
Amp TokenHandler Amp TokenHandler
SingleQuote TokenHandler SingleQuote TokenHandler
RoundOpen TokenHandler RoundOpen TokenHandler
LeftParen TokenHandler LeftParen TokenHandler
RoundClose TokenHandler RoundClose TokenHandler
RightParen TokenHandler RightParen TokenHandler
Asterisk TokenHandler Asterisk TokenHandler
Multiply TokenHandler Multiply TokenHandler
Plus TokenHandler Plus TokenHandler
Add TokenHandler Add TokenHandler
Comma TokenHandler Comma TokenHandler
Minus TokenHandler Minus TokenHandler
Subtract TokenHandler Subtract TokenHandler
Dot TokenHandler Dot TokenHandler
Slash TokenHandler Slash TokenHandler
Divide TokenHandler Divide TokenHandler
Colon TokenHandler Colon TokenHandler
Semicolon TokenHandler Semicolon TokenHandler
AngleOpen TokenHandler AngleOpen TokenHandler
LessThan TokenHandler LessThan TokenHandler
Equal TokenHandler Equal TokenHandler
AngleClose TokenHandler AngleClose TokenHandler
GreaterThan TokenHandler GreaterThan TokenHandler
Question TokenHandler Question TokenHandler
At TokenHandler At TokenHandler
SquareOpen TokenHandler SquareOpen TokenHandler
Backslash TokenHandler Backslash TokenHandler
SquareClose TokenHandler SquareClose TokenHandler
Caret TokenHandler Caret TokenHandler
Underscore TokenHandler Underscore TokenHandler
Backquote TokenHandler Backquote TokenHandler
CurlyOpen TokenHandler CurlyOpen TokenHandler
Pipe TokenHandler Pipe TokenHandler
CurlyClose TokenHandler CurlyClose TokenHandler
Tilde TokenHandler Tilde TokenHandler
Newline TokenHandler Newline TokenHandler
Whitespace TokenHandler Blank TokenHandler
WhitespaceAndNewlines TokenHandler Blanks TokenHandler
EndOfLine TokenHandler Whitespace TokenHandler
Digit TokenHandler EndOfLine TokenHandler
DigitNotZero TokenHandler Digit TokenHandler
Digits TokenHandler DigitNotZero TokenHandler
Float TokenHandler Digits TokenHandler
Boolean TokenHandler Float TokenHandler
Integer TokenHandler Boolean TokenHandler
Signed func(TokenHandler) TokenHandler Integer TokenHandler
IntegerBetween func(min int64, max int64) TokenHandler Signed func(TokenHandler) TokenHandler
ASCII TokenHandler IntegerBetween func(min int64, max int64) TokenHandler
ASCIILower TokenHandler ASCII TokenHandler
ASCIIUpper TokenHandler ASCIILower TokenHandler
HexDigit TokenHandler ASCIIUpper TokenHandler
Octet TokenHandler Letter TokenHandler
IPv4 TokenHandler Lower TokenHandler
IPv4CIDRMask TokenHandler Upper TokenHandler
IPv4Netmask TokenHandler HexDigit TokenHandler
IPv4Net TokenHandler Octet TokenHandler
IPv6 TokenHandler IPv4 TokenHandler
IPv6CIDRMask TokenHandler IPv4CIDRMask TokenHandler
IPv6Net TokenHandler IPv4Netmask TokenHandler
IPv4Net TokenHandler
IPv6 TokenHandler
IPv6CIDRMask TokenHandler
IPv6Net TokenHandler
}{ }{
Rune: MatchRune, Rune: MatchRune,
Runes: MatchRunes, Runes: MatchRunes,
RuneRange: MatchRuneRange, RuneRange: MatchRuneRange,
Str: MatchStr, Str: MatchStr,
StrNoCase: MatchStrNoCase, StrNoCase: MatchStrNoCase,
EndOfFile: MatchEndOfFile(), EndOfFile: MatchEndOfFile(),
AnyRune: MatchAnyRune(), AnyRune: MatchAnyRune(),
ValidRune: MatchValidRune(), ValidRune: MatchValidRune(),
Space: MatchRune(' '), Space: MatchRune(' '),
Tab: MatchRune('\t'), Tab: MatchRune('\t'),
CR: MatchRune('\r'), CR: MatchRune('\r'),
LF: MatchRune('\n'), LF: MatchRune('\n'),
CRLF: MatchStr("\r\n"), CRLF: MatchStr("\r\n"),
Excl: MatchRune('!'), Excl: MatchRune('!'),
DoubleQuote: MatchRune('"'), DoubleQuote: MatchRune('"'),
Hash: MatchRune('#'), Hash: MatchRune('#'),
Dollar: MatchRune('$'), Dollar: MatchRune('$'),
Percent: MatchRune('%'), Percent: MatchRune('%'),
Amp: MatchRune('&'), Amp: MatchRune('&'),
SingleQuote: MatchRune('\''), SingleQuote: MatchRune('\''),
RoundOpen: MatchRune('('), RoundOpen: MatchRune('('),
LeftParen: MatchRune('('), LeftParen: MatchRune('('),
RoundClose: MatchRune(')'), RoundClose: MatchRune(')'),
RightParen: MatchRune(')'), RightParen: MatchRune(')'),
Asterisk: MatchRune('*'), Asterisk: MatchRune('*'),
Multiply: MatchRune('*'), Multiply: MatchRune('*'),
Plus: MatchRune('+'), Plus: MatchRune('+'),
Add: MatchRune('+'), Add: MatchRune('+'),
Comma: MatchRune(','), Comma: MatchRune(','),
Minus: MatchRune('-'), Minus: MatchRune('-'),
Subtract: MatchRune('-'), Subtract: MatchRune('-'),
Dot: MatchRune('.'), Dot: MatchRune('.'),
Slash: MatchRune('/'), Slash: MatchRune('/'),
Divide: MatchRune('/'), Divide: MatchRune('/'),
Colon: MatchRune(':'), Colon: MatchRune(':'),
Semicolon: MatchRune(';'), Semicolon: MatchRune(';'),
AngleOpen: MatchRune('<'), AngleOpen: MatchRune('<'),
LessThan: MatchRune('<'), LessThan: MatchRune('<'),
Equal: MatchRune('='), Equal: MatchRune('='),
AngleClose: MatchRune('>'), AngleClose: MatchRune('>'),
GreaterThan: MatchRune('>'), GreaterThan: MatchRune('>'),
Question: MatchRune('?'), Question: MatchRune('?'),
At: MatchRune('@'), At: MatchRune('@'),
SquareOpen: MatchRune('['), SquareOpen: MatchRune('['),
Backslash: MatchRune('\\'), Backslash: MatchRune('\\'),
SquareClose: MatchRune(']'), SquareClose: MatchRune(']'),
Caret: MatchRune('^'), Caret: MatchRune('^'),
Underscore: MatchRune('_'), Underscore: MatchRune('_'),
Backquote: MatchRune('`'), Backquote: MatchRune('`'),
CurlyOpen: MatchRune('{'), CurlyOpen: MatchRune('{'),
Pipe: MatchRune('|'), Pipe: MatchRune('|'),
CurlyClose: MatchRune('}'), CurlyClose: MatchRune('}'),
Tilde: MatchRune('~'), Tilde: MatchRune('~'),
Whitespace: MatchWhitespace(), Blank: MatchBlank(),
WhitespaceAndNewlines: MatchWhitespaceAndNewlines(), Blanks: MatchBlanks(),
EndOfLine: MatchEndOfLine(), Whitespace: MatchWhitespace(),
Digit: MatchDigit(), EndOfLine: MatchEndOfLine(),
DigitNotZero: MatchDigitNotZero(), Digit: MatchDigit(),
Digits: MatchDigits(), DigitNotZero: MatchDigitNotZero(),
Integer: MatchInteger(), Digits: MatchDigits(),
Signed: MatchSigned, Integer: MatchInteger(),
IntegerBetween: MatchIntegerBetween, Signed: MatchSigned,
Float: MatchFloat(), IntegerBetween: MatchIntegerBetween,
Boolean: MatchBoolean(), Float: MatchFloat(),
ASCII: MatchASCII(), Boolean: MatchBoolean(),
ASCIILower: MatchASCIILower(), ASCII: MatchASCII(),
ASCIIUpper: MatchASCIIUpper(), ASCIILower: MatchASCIILower(),
HexDigit: MatchHexDigit(), ASCIIUpper: MatchASCIIUpper(),
Octet: MatchOctet(false), Letter: MatchUnicodeLetter(),
IPv4: MatchIPv4(true), Lower: MatchUnicodeLower(),
IPv4CIDRMask: MatchIPv4CIDRMask(true), Upper: MatchUnicodeUpper(),
IPv4Netmask: MatchIPv4Netmask(true), HexDigit: MatchHexDigit(),
IPv4Net: MatchIPv4Net(true), Octet: MatchOctet(false),
IPv6: MatchIPv6(true), IPv4: MatchIPv4(true),
IPv6CIDRMask: MatchIPv6CIDRMask(true), IPv4CIDRMask: MatchIPv4CIDRMask(true),
IPv6Net: MatchIPv6Net(true), IPv4Netmask: MatchIPv4Netmask(true),
IPv4Net: MatchIPv4Net(true),
IPv6: MatchIPv6(true),
IPv6CIDRMask: MatchIPv6CIDRMask(true),
IPv6Net: MatchIPv6Net(true),
} }
// M provides convenient access to a range of modifiers (which in their nature are // M provides convenient access to a range of modifiers (which in their nature are
@ -352,20 +360,45 @@ func MatchRuneRange(start rune, end rune) TokenHandler {
} }
} }
// MatchWhitespace creates a TokenHandler that matches the input against one // MatchBlank creates a TokenHandler that matches one rune from the input
// or more whitespace characters, meansing tabs and spaces. // against blank characters, meaning tabs and spaces.
// //
// When you need whitespace matching to also include newlines, then make use // When you need whitespace matching, which also includes characters like
// of MatchWhitespaceAndNewlines(). // newlines, then take a look at MatchWhitespace().
func MatchWhitespace() TokenHandler { func MatchBlank() TokenHandler {
return MatchOneOrMore(MatchAny(MatchRune(' '), MatchRune('\t'))) return MatchAny(MatchRune(' '), MatchRune('\t'))
} }
// MatchWhitespaceAndNewlines creates a TokenHandler that matches the input // MatchBlanks creates a TokenHandler that matches the input against one
// against one or more whitespace and/or newline characters, meaning tabs, // or more blank characters, meaning tabs and spaces.
// spaces and newlines ("\r\n" and "\n"). //
func MatchWhitespaceAndNewlines() TokenHandler { // When you need whitespace matching, which also includes characters like
return MatchOneOrMore(MatchAny(MatchRune(' '), MatchRune('\t'), MatchStr("\r\n"), MatchRune('\n'))) // newlines, then make use of MatchSpace().
func MatchBlanks() TokenHandler {
return MatchOneOrMore(MatchBlank())
}
// MatchWhitespace creates a TokenHandler that matches the input against one or more
// whitespace characters, as defined by unicode.
func MatchWhitespace() TokenHandler {
return MatchOneOrMore(MatchRuneByCallback(unicode.IsSpace))
}
// MatchRuneByCallback creates a TokenHandler that matches a single rune from the
// input against the provided callback function. When the callback returns true,
// it is considered a match.
//
// Note that the callback function matches the signature of the unicode.Is* functions,
// so those can be used. E.g. MatchRuneByCallback(unicode.IsLower).
func MatchRuneByCallback(callback func(rune) bool) TokenHandler {
return func(t *TokenAPI) bool {
input, err := t.NextRune()
if err == nil && callback(input) {
t.Accept()
return true
}
return false
}
} }
// MatchEndOfLine creates a TokenHandler that matches a newline ("\r\n" or "\n") or EOF. // MatchEndOfLine creates a TokenHandler that matches a newline ("\r\n" or "\n") or EOF.
@ -649,7 +682,7 @@ func MatchDigit() TokenHandler {
// MatchDigits creates a TokenHandler that checks if one or more digits can be read // MatchDigits creates a TokenHandler that checks if one or more digits can be read
// from the input. // from the input.
func MatchDigits() TokenHandler { func MatchDigits() TokenHandler {
return MatchOneOrMore(MatchRuneRange('0', '9')) return MatchOneOrMore(MatchDigit())
} }
// MatchDigitNotZero creates a TokenHandler that checks if a single digit not equal // MatchDigitNotZero creates a TokenHandler that checks if a single digit not equal
@ -707,6 +740,24 @@ func MatchASCIIUpper() TokenHandler {
return MatchRuneRange('A', 'Z') return MatchRuneRange('A', 'Z')
} }
// MatchUnicodeLetter creates a TokenHandler function that matches against any
// unicode letter on the input (see unicode.IsLetter(rune)).
func MatchUnicodeLetter() TokenHandler {
return MatchRuneByCallback(unicode.IsLetter)
}
// MatchUnicodeUpper creates a TokenHandler function that matches against any
// upper case unicode letter on the input (see unicode.IsUpper(rune)).
func MatchUnicodeUpper() TokenHandler {
return MatchRuneByCallback(unicode.IsUpper)
}
// MatchUnicodeLower creates a TokenHandler function that matches against any
// lower case unicode letter on the input (see unicode.IsLower(rune)).
func MatchUnicodeLower() TokenHandler {
return MatchRuneByCallback(unicode.IsLower)
}
// MatchHexDigit creates a TokenHandler function that check if a single hexadecimal // MatchHexDigit creates a TokenHandler function that check if a single hexadecimal
// digit can be read from the input. // digit can be read from the input.
func MatchHexDigit() TokenHandler { func MatchHexDigit() TokenHandler {
@ -908,15 +959,15 @@ func MatchIPv6Net(normalize bool) TokenHandler {
// //
// Note that if the TokenHandler does not apply, a mismatch will be reported back, // Note that if the TokenHandler does not apply, a mismatch will be reported back,
// even though we would have dropped the output anyway. So if you would like // even though we would have dropped the output anyway. So if you would like
// to drop optional whitespace, then use something like: // to drop optional blanks (spaces and tabs), then use something like:
// //
// M.Drop(C.Opt(A.Whitespace)) // M.Drop(C.Opt(A.Blank))
// //
// instead of: // instead of:
// //
// M.Drop(A.Whitespace) // M.Drop(A.Blank)
// //
// Since whitespace is defined as "1 or more spaces and/or tabs", the input // Since A.Blanks is defined as "1 or more spaces and/or tabs", the input
// string "bork" would not match against the second form, but " bork" would. // string "bork" would not match against the second form, but " bork" would.
// In both cases, it would match the first form. // In both cases, it would match the first form.
func ModifyDrop(handler TokenHandler) TokenHandler { func ModifyDrop(handler TokenHandler) TokenHandler {
@ -960,8 +1011,8 @@ func modifyTrim(handler TokenHandler, cutset string, trimLeft bool, trimRight bo
} }
// ModifyTrimSpace creates a TokenHandler that checks if the provided TokenHandler applies. // ModifyTrimSpace creates a TokenHandler that checks if the provided TokenHandler applies.
// If it does, then its output is taken and all leading and trailing whitespace charcters, // If it does, then its output is taken and all leading and trailing whitespace characters,
// as defined by Unicode (spaces, tabs, carriage returns and newlines) are removed from it. // as defined by Unicode are removed from it.
func ModifyTrimSpace(handler TokenHandler) TokenHandler { func ModifyTrimSpace(handler TokenHandler) TokenHandler {
return ModifyByCallback(handler, strings.TrimSpace) return ModifyByCallback(handler, strings.TrimSpace)
} }

22
tokenhandlers_builtin_test.go
View File

@ -157,9 +157,12 @@ func TestAtoms(t *testing.T) {
{"|", a.Pipe, true, "|"}, {"|", a.Pipe, true, "|"},
{"}", a.CurlyClose, true, "}"}, {"}", a.CurlyClose, true, "}"},
{"~", a.Tilde, true, "~"}, {"~", a.Tilde, true, "~"},
{" \t \t \r\n", a.Whitespace, true, " \t \t "}, {"\t \t \r\n", a.Blank, true, "\t"},
{"\r", a.WhitespaceAndNewlines, false, ""}, {" \t \t \r\n", a.Blanks, true, " \t \t "},
{" \t\r\n \r", a.WhitespaceAndNewlines, true, " \t\r\n "}, {"xxx", a.Whitespace, false, ""},
{" ", a.Whitespace, true, " "},
{"\t", a.Whitespace, true, "\t"},
{" \t\r\n \r\v\f ", a.Whitespace, true, " \t\r\n \r\v\f "},
{"", a.EndOfLine, true, ""}, {"", a.EndOfLine, true, ""},
{"\r\n", a.EndOfLine, true, "\r\n"}, {"\r\n", a.EndOfLine, true, "\r\n"},
{"\n", a.EndOfLine, true, "\n"}, {"\n", a.EndOfLine, true, "\n"},
@ -182,6 +185,13 @@ func TestAtoms(t *testing.T) {
{"Z", a.ASCIIUpper, true, "Z"}, {"Z", a.ASCIIUpper, true, "Z"},
{"a", a.ASCIIUpper, false, ""}, {"a", a.ASCIIUpper, false, ""},
{"z", a.ASCIIUpper, false, ""}, {"z", a.ASCIIUpper, false, ""},
{"1", a.Letter, false, ""},
{"a", a.Letter, true, "a"},
{"Ø", a.Letter, true, "Ø"},
{"Ë", a.Lower, false, ""},
{"ë", a.Lower, true, "ë"},
{"ä", a.Upper, false, "ä"},
{"Ä", a.Upper, true, "Ä"},
{"0", a.HexDigit, true, "0"}, {"0", a.HexDigit, true, "0"},
{"9", a.HexDigit, true, "9"}, {"9", a.HexDigit, true, "9"},
{"a", a.HexDigit, true, "a"}, {"a", a.HexDigit, true, "a"},
@ -403,16 +413,16 @@ func TestCombination(t *testing.T) {
c.Opt(a.SquareOpen), c.Opt(a.SquareOpen),
m.Trim( m.Trim(
c.Seq( c.Seq(
c.Opt(a.Whitespace), c.Opt(a.Blanks),
c.Rep(3, a.AngleClose), c.Rep(3, a.AngleClose),
m.ByCallback(c.OneOrMore(a.StrNoCase("hello")), func(s string) string { m.ByCallback(c.OneOrMore(a.StrNoCase("hello")), func(s string) string {
return fmt.Sprintf("%d", len(s)) return fmt.Sprintf("%d", len(s))
}), }),
m.Replace(c.Separated(a.Comma, c.Opt(a.Whitespace)), ", "), m.Replace(c.Separated(a.Comma, c.Opt(a.Blanks)), ", "),
m.ToUpper(c.Min(1, a.ASCIILower)), m.ToUpper(c.Min(1, a.ASCIILower)),
m.Drop(a.Excl), m.Drop(a.Excl),
c.Rep(3, a.AngleOpen), c.Rep(3, a.AngleOpen),
c.Opt(a.Whitespace), c.Opt(a.Blanks),
), ),
" \t", " \t",
), ),