OpenSourceProjects
/
go-parsekit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Moving output functions to its own substruct of the API.

This commit is contained in:
Maurice Makaay 2019-07-19 22:57:06 +00:00
parent 458d6f60a6
commit 9d98c9dff7
7 changed files with 148 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
parse/api.go
View File

@ -29,8 +29,8 @@ type API struct {
func (p *API) Peek(tokenHandler tokenize.Handler) bool { func (p *API) Peek(tokenHandler tokenize.Handler) bool {
forkedAPI, ok := p.invokeHandler("Peek", tokenHandler) forkedAPI, ok := p.invokeHandler("Peek", tokenHandler)
if ok { if ok {
p.Result.Tokens = p.tokenAPI.Tokens() p.Result.Tokens = p.tokenAPI.Output.Tokens()
p.Result.Runes = p.tokenAPI.Runes() p.Result.Runes = p.tokenAPI.Output.Runes()
} }
p.tokenAPI.Dispose(forkedAPI) p.tokenAPI.Dispose(forkedAPI)
return ok return ok
@ -46,8 +46,8 @@ func (p *API) Accept(tokenHandler tokenize.Handler) bool {
forkedAPI, ok := p.invokeHandler("Accept", tokenHandler) forkedAPI, ok := p.invokeHandler("Accept", tokenHandler)
if ok { if ok {
// Keep track of the results. // Keep track of the results.
p.Result.Tokens = p.tokenAPI.Tokens() p.Result.Tokens = p.tokenAPI.Output.Tokens()
p.Result.Runes = p.tokenAPI.Runes() p.Result.Runes = p.tokenAPI.Output.Runes()
// Merge to the parent level. // Merge to the parent level.
p.tokenAPI.Merge(forkedAPI) p.tokenAPI.Merge(forkedAPI)

113
tokenize/api.go
View File

@ -78,6 +78,7 @@ type API struct {
stackFrames []stackFrame // the stack frames, containing stack level-specific data stackFrames []stackFrame // the stack frames, containing stack level-specific data
stackLevel int // the current stack level stackLevel int // the current stack level
stackFrame *stackFrame // the current stack frame stackFrame *stackFrame // the current stack frame
Output Output // provides output-related functionality
} }
type stackFrame struct { type stackFrame struct {
@ -93,6 +94,11 @@ type stackFrame struct {
err error // can be used by a Handler to report a specific issue with the input err error // can be used by a Handler to report a specific issue with the input
} }
// Output provides output-related functionality for the tokenize API.
type Output struct {
api *API
}
const initialStackDepth = 64 const initialStackDepth = 64
const initialTokenStoreLength = 64 const initialTokenStoreLength = 64
const initialByteStoreLength = 1024 const initialByteStoreLength = 1024
@ -107,6 +113,7 @@ func NewAPI(input interface{}) *API {
tokens: make([]Token, initialTokenStoreLength), tokens: make([]Token, initialTokenStoreLength),
stackFrames: make([]stackFrame, initialStackDepth), stackFrames: make([]stackFrame, initialStackDepth),
} }
api.Output = Output{api: api}
api.stackFrame = &api.stackFrames[0] api.stackFrame = &api.stackFrames[0]
return api return api
@ -460,73 +467,69 @@ func (i *API) FlushInput() bool {
return false return false
} }
func (i *API) String() string { func (o Output) String() string {
bytes := i.bytes[i.stackFrame.bytesStart:i.stackFrame.bytesEnd] bytes := o.api.bytes[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd]
return string(bytes) return string(bytes)
} }
func (i *API) Runes() []rune { func (o Output) Runes() []rune {
bytes := i.bytes[i.stackFrame.bytesStart:i.stackFrame.bytesEnd] bytes := o.api.bytes[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd]
return []rune(string(bytes)) return []rune(string(bytes))
} }
func (i *API) Rune(offset int) rune { func (o Output) Rune(offset int) rune {
r, _ := utf8.DecodeRune(i.bytes[i.stackFrame.bytesStart+offset:]) r, _ := utf8.DecodeRune(o.api.bytes[o.api.stackFrame.bytesStart+offset:])
return r return r
} }
func (i *API) ClearBytes() { func (o Output) ClearData() {
i.stackFrame.bytesEnd = i.stackFrame.bytesStart o.api.stackFrame.bytesEnd = o.api.stackFrame.bytesStart
} }
func (i *API) SetBytes(bytes ...byte) { func (o Output) SetBytes(bytes ...byte) {
i.ClearBytes() o.ClearData()
i.AddBytes(bytes...) o.AddBytes(bytes...)
} }
func (i *API) AddBytes(bytes ...byte) { func (o Output) AddBytes(bytes ...byte) {
// Grow the runes capacity when needed. // Grow the runes capacity when needed.
newBytesEnd := i.stackFrame.bytesEnd + len(bytes) newBytesEnd := o.api.stackFrame.bytesEnd + len(bytes)
if cap(i.bytes) < newBytesEnd { if cap(o.api.bytes) < newBytesEnd {
newBytes := make([]byte, newBytesEnd*2) newBytes := make([]byte, newBytesEnd*2)
copy(newBytes, i.bytes) copy(newBytes, o.api.bytes)
i.bytes = newBytes o.api.bytes = newBytes
} }
copy(i.bytes[i.stackFrame.bytesEnd:], bytes) copy(o.api.bytes[o.api.stackFrame.bytesEnd:], bytes)
i.stackFrame.bytesEnd = newBytesEnd o.api.stackFrame.bytesEnd = newBytesEnd
} }
func (i *API) ClearRunes() { func (o Output) SetRunes(runes ...rune) {
i.stackFrame.bytesEnd = i.stackFrame.bytesStart o.ClearData()
o.AddRunes(runes...)
} }
func (i *API) SetRunes(runes ...rune) { func (o Output) AddRunes(runes ...rune) {
i.ClearRunes()
i.AddRunes(runes...)
}
func (i *API) AddRunes(runes ...rune) {
// Grow the runes capacity when needed. // Grow the runes capacity when needed.
runesAsString := string(runes) runesAsString := string(runes)
newBytesEnd := i.stackFrame.bytesEnd + len(runesAsString) newBytesEnd := o.api.stackFrame.bytesEnd + len(runesAsString)
if cap(i.bytes) < newBytesEnd { if cap(o.api.bytes) < newBytesEnd {
newBytes := make([]byte, newBytesEnd*2) newBytes := make([]byte, newBytesEnd*2)
copy(newBytes, i.bytes) copy(newBytes, o.api.bytes)
i.bytes = newBytes o.api.bytes = newBytes
} }
copy(i.bytes[i.stackFrame.bytesEnd:], runesAsString) copy(o.api.bytes[o.api.stackFrame.bytesEnd:], runesAsString)
i.stackFrame.bytesEnd = newBytesEnd o.api.stackFrame.bytesEnd = newBytesEnd
} }
func (i *API) AddString(s string) { func (o Output) AddString(s string) {
i.AddBytes([]byte(s)...) o.AddBytes([]byte(s)...)
} }
func (i *API) SetString(s string) { func (o Output) SetString(s string) {
i.ClearBytes() o.ClearData()
i.SetBytes([]byte(s)...) o.SetBytes([]byte(s)...)
} }
func (i *API) Cursor() string { func (i *API) Cursor() string {
@ -536,38 +539,38 @@ func (i *API) Cursor() string {
return fmt.Sprintf("line %d, column %d", i.stackFrame.line+1, i.stackFrame.column+1) return fmt.Sprintf("line %d, column %d", i.stackFrame.line+1, i.stackFrame.column+1)
} }
func (i *API) Tokens() []Token { func (o Output) Tokens() []Token {
return i.tokens[i.stackFrame.tokenStart:i.stackFrame.tokenEnd] return o.api.tokens[o.api.stackFrame.tokenStart:o.api.stackFrame.tokenEnd]
} }
func (i *API) Token(offset int) Token { func (o Output) Token(offset int) Token {
return i.tokens[i.stackFrame.tokenStart+offset] return o.api.tokens[o.api.stackFrame.tokenStart+offset]
} }
func (i *API) TokenValue(offset int) interface{} { func (o Output) TokenValue(offset int) interface{} {
return i.tokens[i.stackFrame.tokenStart+offset].Value return o.api.tokens[o.api.stackFrame.tokenStart+offset].Value
} }
func (i *API) ClearTokens() { func (o Output) ClearTokens() {
i.stackFrame.tokenEnd = i.stackFrame.tokenStart o.api.stackFrame.tokenEnd = o.api.stackFrame.tokenStart
} }
func (i *API) SetTokens(tokens ...Token) { func (o Output) SetTokens(tokens ...Token) {
i.ClearTokens() o.ClearTokens()
i.AddTokens(tokens...) o.AddTokens(tokens...)
} }
func (i *API) AddTokens(tokens ...Token) { func (o Output) AddTokens(tokens ...Token) {
// Grow the tokens capacity when needed. // Grow the tokens capacity when needed.
newTokenEnd := i.stackFrame.tokenEnd + len(tokens) newTokenEnd := o.api.stackFrame.tokenEnd + len(tokens)
if cap(i.tokens) < newTokenEnd { if cap(o.api.tokens) < newTokenEnd {
newTokens := make([]Token, newTokenEnd*2) newTokens := make([]Token, newTokenEnd*2)
copy(newTokens, i.tokens) copy(newTokens, o.api.tokens)
i.tokens = newTokens o.api.tokens = newTokens
} }
for offset, t := range tokens { for offset, t := range tokens {
i.tokens[i.stackFrame.tokenEnd+offset] = t o.api.tokens[o.api.stackFrame.tokenEnd+offset] = t
} }
i.stackFrame.tokenEnd = newTokenEnd o.api.stackFrame.tokenEnd = newTokenEnd
} }

92
tokenize/api_test.go
View File

@ -19,7 +19,7 @@ func ExampleNewAPI() {
// r, err := api.NextRune() // r, err := api.NextRune()
// fmt.Printf("Rune read from input; %c\n", r) // fmt.Printf("Rune read from input; %c\n", r)
// fmt.Printf("The error: %v\n", err) // fmt.Printf("The error: %v\n", err)
// fmt.Printf("API results: %q\n", api.String()) // fmt.Printf("API results: %q\n", api.Output.String())
// // Output: // // Output:
// // Rune read from input; T // // Rune read from input; T
@ -55,7 +55,7 @@ func ExampleAPI_AcceptRune() {
// Reads 'e', but does not accept it to the API results. // Reads 'e', but does not accept it to the API results.
r, _, _ = api.PeekRune(0) r, _, _ = api.PeekRune(0)
fmt.Printf("API results: %q\n", api.String()) fmt.Printf("API results: %q\n", api.Output.String())
// Output: // Output:
// API results: "Th" // API results: "Th"
@ -74,7 +74,7 @@ func ExampleAPI_AcceptRunes() {
// Accepts only 'T' and 'h' into the API results. // Accepts only 'T' and 'h' into the API results.
api.AcceptRunes(r0, r1) api.AcceptRunes(r0, r1)
fmt.Printf("API results: %q\n", api.String()) fmt.Printf("API results: %q\n", api.Output.String())
// Output: // Output:
// API results: "Th" // API results: "Th"
@ -99,7 +99,7 @@ func ExampleAPI_SkipRune() {
} }
} }
fmt.Printf("API results: %q\n", api.String()) fmt.Printf("API results: %q\n", api.Output.String())
// Output: // Output:
// API results: "eiuaeAIiae" // API results: "eiuaeAIiae"
@ -108,29 +108,29 @@ func ExampleAPI_SkipRune() {
func ExampleAPI_modifyingResults() { func ExampleAPI_modifyingResults() {
api := tokenize.NewAPI("") api := tokenize.NewAPI("")
api.AddString("Some runes") api.Output.AddString("Some runes")
api.AddRunes(' ', 'a', 'd', 'd', 'e', 'd') api.Output.AddRunes(' ', 'a', 'd', 'd', 'e', 'd')
api.AddRunes(' ', 'i', 'n', ' ') api.Output.AddRunes(' ', 'i', 'n', ' ')
api.AddString("various ways") api.Output.AddString("various ways")
fmt.Printf("API result first 10 runes: %q\n", api.Runes()[0:10]) fmt.Printf("API result first 10 runes: %q\n", api.Output.Runes()[0:10])
fmt.Printf("API result runes as string: %q\n", api.String()) fmt.Printf("API result runes as string: %q\n", api.Output.String())
api.SetString("new ") api.Output.SetString("new ")
api.AddString("set ") api.Output.AddString("set ")
api.AddString("of ") api.Output.AddString("of ")
api.AddRunes('r', 'u', 'n', 'e', 's') api.Output.AddRunes('r', 'u', 'n', 'e', 's')
fmt.Printf("API result runes as string: %q\n", api.String()) fmt.Printf("API result runes as string: %q\n", api.Output.String())
fmt.Printf("API result runes: %q\n", api.Runes()) fmt.Printf("API result runes: %q\n", api.Output.Runes())
fmt.Printf("API third rune: %q\n", api.Rune(2)) fmt.Printf("API third rune: %q\n", api.Output.Rune(2))
api.AddTokens(tokenize.Token{ api.Output.AddTokens(tokenize.Token{
Type: 42, Type: 42,
Value: "towel"}) Value: "towel"})
api.AddTokens(tokenize.Token{ api.Output.AddTokens(tokenize.Token{
Type: 73, Type: 73,
Value: "Zaphod"}) Value: "Zaphod"})
fmt.Printf("API result tokens: %v\n", api.Tokens()) fmt.Printf("API result tokens: %v\n", api.Output.Tokens())
fmt.Printf("API second result token: %v\n", api.Token(1)) fmt.Printf("API second result token: %v\n", api.Output.Token(1))
// Output: // Output:
// API result first 10 runes: ['S' 'o' 'm' 'e' ' ' 'r' 'u' 'n' 'e' 's'] // API result first 10 runes: ['S' 'o' 'm' 'e' ' ' 'r' 'u' 'n' 'e' 's']
@ -149,18 +149,18 @@ func ExampleAPI_Reset() {
api.AcceptRune(r) api.AcceptRune(r)
r, _, _ = api.PeekRune(0) // read 'e' r, _, _ = api.PeekRune(0) // read 'e'
api.AcceptRune(r) api.AcceptRune(r)
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor())
// Reset clears the results. // Reset clears the results.
api.Reset() api.Reset()
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor())
// So then doing the same read operations, the same data are read. // So then doing the same read operations, the same data are read.
r, _, _ = api.PeekRune(0) // read 'V' r, _, _ = api.PeekRune(0) // read 'V'
api.AcceptRune(r) api.AcceptRune(r)
r, _, _ = api.PeekRune(0) // read 'e' r, _, _ = api.PeekRune(0) // read 'e'
api.AcceptRune(r) api.AcceptRune(r)
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor()) fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor())
// Output: // Output:
// API results: "Ve" at line 1, column 3 // API results: "Ve" at line 1, column 3
@ -246,23 +246,23 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
r, _, _ := api.PeekRune(0) r, _, _ := api.PeekRune(0)
AssertEqual(t, 'a', r, "child4 rune 1") AssertEqual(t, 'a', r, "child4 rune 1")
api.AcceptRune(r) api.AcceptRune(r)
AssertEqual(t, "a", api.String(), "child4 runes after rune 1") AssertEqual(t, "a", api.Output.String(), "child4 runes after rune 1")
// Read another rune 'b' from child4. // Read another rune 'b' from child4.
r, _, _ = api.PeekRune(0) r, _, _ = api.PeekRune(0)
AssertEqual(t, 'b', r, "child4 rune 2") AssertEqual(t, 'b', r, "child4 rune 2")
api.AcceptRune(r) api.AcceptRune(r)
AssertEqual(t, "ab", api.String(), "child4 runes after rune 2") AssertEqual(t, "ab", api.Output.String(), "child4 runes after rune 2")
// Merge "ab" from child4 to child3. // Merge "ab" from child4 to child3.
api.Merge(child4) api.Merge(child4)
AssertEqual(t, "", api.String(), "child4 runes after first merge") AssertEqual(t, "", api.Output.String(), "child4 runes after first merge")
// Read some more from child4. // Read some more from child4.
r, _, _ = api.PeekRune(0) r, _, _ = api.PeekRune(0)
AssertEqual(t, 'c', r, "child4 rune 3") AssertEqual(t, 'c', r, "child4 rune 3")
api.AcceptRune(r) api.AcceptRune(r)
AssertEqual(t, "c", api.String(), "child4 runes after rune 1") AssertEqual(t, "c", api.Output.String(), "child4 runes after rune 1")
AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child4 rune 3") AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child4 rune 3")
// Merge "c" from child4 to child3. // Merge "c" from child4 to child3.
@ -272,7 +272,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
api.Dispose(child4) api.Dispose(child4)
// Child3 should now have the compbined results "abc" from child4's work. // Child3 should now have the compbined results "abc" from child4's work.
AssertEqual(t, "abc", api.String(), "child3 after merge of child4") AssertEqual(t, "abc", api.Output.String(), "child3 after merge of child4")
AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child3 rune 3, after merge of child4") AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child3 rune 3, after merge of child4")
// Now read some data from child3. // Now read some data from child3.
@ -288,7 +288,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
AssertEqual(t, 'f', r, "child3 rune 5") AssertEqual(t, 'f', r, "child3 rune 5")
api.AcceptRune(r) api.AcceptRune(r)
AssertEqual(t, "abcdef", api.String(), "child3 total result after rune 6") AssertEqual(t, "abcdef", api.Output.String(), "child3 total result after rune 6")
// Temporarily go some new forks from here, but don't use their outcome. // Temporarily go some new forks from here, but don't use their outcome.
child3sub1 := api.Fork() child3sub1 := api.Fork()
@ -308,7 +308,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
api.Merge(child3) api.Merge(child3)
api.Dispose(child3) api.Dispose(child3)
AssertEqual(t, "abcdef", api.String(), "child2 total result after merge of child3") AssertEqual(t, "abcdef", api.Output.String(), "child2 total result after merge of child3")
AssertEqual(t, "line 1, column 7", api.Cursor(), "cursor child2 after merge child3") AssertEqual(t, "line 1, column 7", api.Cursor(), "cursor child2 after merge child3")
// Merge child2 to child1 and dispose of it. // Merge child2 to child1 and dispose of it.
@ -328,23 +328,23 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
r, _, _ = api.PeekRune(0) r, _, _ = api.PeekRune(0)
api.AcceptRune(r) api.AcceptRune(r)
AssertEqual(t, "abcdefg", api.String(), "api string end result") AssertEqual(t, "abcdefg", api.Output.String(), "api string end result")
AssertEqual(t, "line 1, column 8", api.Cursor(), "api cursor end result") AssertEqual(t, "line 1, column 8", api.Cursor(), "api cursor end result")
} }
func TestClearRunes(t *testing.T) { func TestClearData(t *testing.T) {
api := tokenize.NewAPI("Laphroaig") api := tokenize.NewAPI("Laphroaig")
r, _, _ := api.PeekRune(0) // Read 'L' r, _, _ := api.PeekRune(0) // Read 'L'
api.AcceptRune(r) // Add to runes api.AcceptRune(r) // Add to runes
r, _, _ = api.PeekRune(0) // Read 'a' r, _, _ = api.PeekRune(0) // Read 'a'
api.AcceptRune(r) // Add to runes api.AcceptRune(r) // Add to runes
api.ClearRunes() // Clear the runes, giving us a fresh start. api.Output.ClearData() // Clear the runes, giving us a fresh start.
r, _, _ = api.PeekRune(0) // Read 'p' r, _, _ = api.PeekRune(0) // Read 'p'
api.AcceptRune(r) // Add to runes api.AcceptRune(r) // Add to runes
r, _, _ = api.PeekRune(0) // Read 'r' r, _, _ = api.PeekRune(0) // Read 'r'
api.AcceptRune(r) // Add to runes api.AcceptRune(r) // Add to runes
AssertEqual(t, "ph", api.String(), "api string end result") AssertEqual(t, "ph", api.Output.String(), "api string end result")
} }
func TestMergeScenariosForTokens(t *testing.T) { func TestMergeScenariosForTokens(t *testing.T) {
@ -355,34 +355,34 @@ func TestMergeScenariosForTokens(t *testing.T) {
token3 := tokenize.Token{Value: 3} token3 := tokenize.Token{Value: 3}
token4 := tokenize.Token{Value: 4} token4 := tokenize.Token{Value: 4}
api.SetTokens(token1) api.Output.SetTokens(token1)
tokens := api.Tokens() tokens := api.Output.Tokens()
AssertEqual(t, 1, len(tokens), "Tokens 1") AssertEqual(t, 1, len(tokens), "Tokens 1")
child := api.Fork() child := api.Fork()
tokens = api.Tokens() tokens = api.Output.Tokens()
AssertEqual(t, 0, len(tokens), "Tokens 2") AssertEqual(t, 0, len(tokens), "Tokens 2")
api.AddTokens(token2) api.Output.AddTokens(token2)
api.Merge(child) api.Merge(child)
api.Dispose(child) api.Dispose(child)
tokens = api.Tokens() tokens = api.Output.Tokens()
AssertEqual(t, 2, len(tokens), "Tokens 3") AssertEqual(t, 2, len(tokens), "Tokens 3")
child = api.Fork() child = api.Fork()
api.AddTokens(token3) api.Output.AddTokens(token3)
api.Reset() api.Reset()
api.AddTokens(token4) api.Output.AddTokens(token4)
api.Merge(child) api.Merge(child)
api.Dispose(child) api.Dispose(child)
tokens = api.Tokens() tokens = api.Output.Tokens()
AssertEqual(t, 3, len(tokens), "Tokens 4") AssertEqual(t, 3, len(tokens), "Tokens 4")
AssertEqual(t, 1, api.TokenValue(0).(int), "Tokens 4, value 0") AssertEqual(t, 1, api.Output.TokenValue(0).(int), "Tokens 4, value 0")
AssertEqual(t, 2, api.TokenValue(1).(int), "Tokens 4, value 1") AssertEqual(t, 2, api.Output.TokenValue(1).(int), "Tokens 4, value 1")
AssertEqual(t, 4, api.TokenValue(2).(int), "Tokens 4, value 2") AssertEqual(t, 4, api.Output.TokenValue(2).(int), "Tokens 4, value 2")
} }

50
tokenize/handlers_builtin.go
View File

@ -987,7 +987,7 @@ func MatchIntegerBetween(min int64, max int64) Handler {
if !digits(t) { if !digits(t) {
return false return false
} }
value, _ := strconv.ParseInt(t.String(), 10, 64) value, _ := strconv.ParseInt(t.Output.String(), 10, 64)
if value < min || value > max { if value < min || value > max {
return false return false
} }
@ -1346,16 +1346,16 @@ func MatchOctet(normalize bool) Handler {
if !max3Digits(t) { if !max3Digits(t) {
return false return false
} }
value, _ := strconv.ParseInt(t.String(), 10, 16) value, _ := strconv.ParseInt(t.Output.String(), 10, 16)
if value > 255 { if value > 255 {
return false return false
} }
if normalize { if normalize {
runes := t.Runes() runes := t.Output.Runes()
for len(runes) > 1 && runes[0] == '0' { for len(runes) > 1 && runes[0] == '0' {
runes = runes[1:] runes = runes[1:]
} }
t.SetRunes(runes...) t.Output.SetRunes(runes...)
} }
return true return true
} }
@ -1397,13 +1397,13 @@ func MatchIPv4Netmask(normalize bool) Handler {
} }
// Check if the mask is provided in canonical form (at the binary level, ones followed by zeroes). // Check if the mask is provided in canonical form (at the binary level, ones followed by zeroes).
mask := net.IPv4Mask(t.TokenValue(0).(byte), t.TokenValue(1).(byte), t.TokenValue(2).(byte), t.TokenValue(3).(byte)) mask := net.IPv4Mask(t.Output.TokenValue(0).(byte), t.Output.TokenValue(1).(byte), t.Output.TokenValue(2).(byte), t.Output.TokenValue(3).(byte))
ones, bits := mask.Size() ones, bits := mask.Size()
if ones == 0 && bits == 0 { if ones == 0 && bits == 0 {
return false return false
} }
t.ClearTokens() t.Output.ClearTokens()
return true return true
} }
} }
@ -1432,18 +1432,18 @@ func MatchIPv4Net(normalize bool) Handler {
return true return true
} }
maskToken := t.Token(1) maskToken := t.Output.Token(1)
if maskToken.Type == "cidr" { if maskToken.Type == "cidr" {
t.SetString(fmt.Sprintf("%s/%d", t.TokenValue(0), t.TokenValue(1).(uint8))) t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), t.Output.TokenValue(1).(uint8)))
} else { } else {
o := strings.Split(t.TokenValue(1).(string), ".") o := strings.Split(t.Output.TokenValue(1).(string), ".")
b := func(idx int) byte { i, _ := strconv.Atoi(o[idx]); return byte(i) } b := func(idx int) byte { i, _ := strconv.Atoi(o[idx]); return byte(i) }
mask := net.IPv4Mask(b(0), b(1), b(2), b(3)) mask := net.IPv4Mask(b(0), b(1), b(2), b(3))
bits, _ := mask.Size() bits, _ := mask.Size()
t.SetString(fmt.Sprintf("%s/%d", t.TokenValue(0), bits)) t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), bits))
} }
t.ClearTokens() t.Output.ClearTokens()
return true return true
} }
} }
@ -1472,13 +1472,13 @@ func MatchIPv6(normalize bool) Handler {
} }
// Invalid IPv6, when net.ParseIP() cannot handle it. // Invalid IPv6, when net.ParseIP() cannot handle it.
parsed := net.ParseIP(t.String()) parsed := net.ParseIP(t.Output.String())
if parsed == nil { if parsed == nil {
return false return false
} }
if normalize { if normalize {
t.SetString(parsed.String()) t.Output.SetString(parsed.String())
} }
return true return true
} }
@ -1501,8 +1501,8 @@ func matchCIDRMask(bits int64, normalize bool) Handler {
if !mask(t) { if !mask(t) {
return false return false
} }
bits, _ := strconv.Atoi(t.String()) bits, _ := strconv.Atoi(t.Output.String())
t.SetString(fmt.Sprintf("%d", bits)) t.Output.SetString(fmt.Sprintf("%d", bits))
return true return true
} }
} }
@ -1643,8 +1643,8 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler {
return func(t *API) bool { return func(t *API) bool {
child := t.Fork() child := t.Fork()
if handler(t) { if handler(t) {
s := modfunc(t.String()) s := modfunc(t.Output.String())
t.SetString(s) t.Output.SetString(s)
t.Merge(child) t.Merge(child)
t.Dispose(child) t.Dispose(child)
return true return true
@ -1661,7 +1661,7 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler {
// an 'n'-character). // an 'n'-character).
func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler { func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler {
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
literal := t.String() literal := t.Output.String()
return literal return literal
}) })
} }
@ -1673,7 +1673,7 @@ func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler {
func MakeStrInterpretedToken(toktype interface{}, handler Handler) Handler { func MakeStrInterpretedToken(toktype interface{}, handler Handler) Handler {
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
// TODO ERROR HANDLING // TODO ERROR HANDLING
interpreted, _ := interpretString(t.String()) interpreted, _ := interpretString(t.Output.String())
return interpreted return interpreted
}) })
} }
@ -1697,7 +1697,7 @@ func interpretString(str string) (string, error) {
func MakeRuneToken(toktype interface{}, handler Handler) Handler { func MakeRuneToken(toktype interface{}, handler Handler) Handler {
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
// TODO ERROR HANDLING --- not a 1 rune input // TODO ERROR HANDLING --- not a 1 rune input
return t.Rune(0) return t.Output.Rune(0)
}) })
} }
@ -1707,7 +1707,7 @@ func MakeRuneToken(toktype interface{}, handler Handler) Handler {
func MakeByteToken(toktype interface{}, handler Handler) Handler { func MakeByteToken(toktype interface{}, handler Handler) Handler {
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
// TODO ERROR HANDLING --- not a 1 byte input // TODO ERROR HANDLING --- not a 1 byte input
return byte(t.Rune(0)) return byte(t.Output.Rune(0))
}) })
} }
@ -1912,7 +1912,7 @@ func MakeBooleanToken(toktype interface{}, handler Handler) Handler {
func makeStrconvToken(name string, toktype interface{}, handler Handler, convert func(s string) (interface{}, error)) Handler { func makeStrconvToken(name string, toktype interface{}, handler Handler, convert func(s string) (interface{}, error)) Handler {
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} { return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
value, err := convert(t.String()) value, err := convert(t.Output.String())
if err != nil { if err != nil {
// TODO meh, panic feels so bad here. Maybe just turn this case into "no match"? // TODO meh, panic feels so bad here. Maybe just turn this case into "no match"?
panic(fmt.Sprintf("%s token invalid (%s)", name, err)) panic(fmt.Sprintf("%s token invalid (%s)", name, err))
@ -1943,7 +1943,7 @@ func MakeTokenByCallback(toktype interface{}, handler Handler, makeValue func(t
// tokens will end up in the order "date", "year", "month", "day". When we'd add the // tokens will end up in the order "date", "year", "month", "day". When we'd add the
// token to the child here, the order would have been "year", "month", "day", "date". // token to the child here, the order would have been "year", "month", "day", "date".
token := Token{Type: toktype, Value: makeValue(t)} token := Token{Type: toktype, Value: makeValue(t)}
t.AddTokens(token) t.Output.AddTokens(token)
t.Merge(child) t.Merge(child)
t.Dispose(child) t.Dispose(child)
@ -1960,10 +1960,10 @@ func MakeTokenGroup(toktype interface{}, handler Handler) Handler {
return func(t *API) bool { return func(t *API) bool {
child := t.Fork() child := t.Fork()
if handler(t) { if handler(t) {
tokens := t.Tokens() tokens := t.Output.Tokens()
tokensCopy := make([]Token, len(tokens)) tokensCopy := make([]Token, len(tokens))
copy(tokensCopy, tokens) copy(tokensCopy, tokens)
t.SetTokens(Token{Type: toktype, Value: tokensCopy}) t.Output.SetTokens(Token{Type: toktype, Value: tokensCopy})
t.Merge(child) t.Merge(child)
t.Dispose(child) t.Dispose(child)
return true return true

4
tokenize/tokenize.go
View File

@ -47,8 +47,8 @@ func New(tokenHandler Handler) Func {
return nil, err return nil, err
} }
result := &Result{ result := &Result{
Runes: api.Runes(), Runes: api.Output.Runes(),
Tokens: api.Tokens(), Tokens: api.Output.Tokens(),
} }
return result, nil return result, nil
} }

2
tokenize/tokenizer_test.go
View File

@ -69,7 +69,7 @@ func TestInputCanAcceptRunesFromReader(t *testing.T) {
r2, _, _ := i.PeekRune(1) r2, _, _ := i.PeekRune(1)
i.AcceptRunes(r1, r2) i.AcceptRunes(r1, r2)
AssertEqual(t, "Tes", i.String(), "i.String()") AssertEqual(t, "Tes", i.Output.String(), "i.String()")
} }
func TestCallingMergeOnTopLevelAPI_Panics(t *testing.T) { func TestCallingMergeOnTopLevelAPI_Panics(t *testing.T) {

24
tokenize/tokenizer_whitebox_test.go
View File

@ -9,7 +9,7 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
i := NewAPI("Testing") i := NewAPI("Testing")
r, _, _ := i.PeekRune(0) r, _, _ := i.PeekRune(0)
i.AcceptRune(r) // T i.AcceptRune(r) // T
AssertEqual(t, "T", i.String(), "accepted rune in input") AssertEqual(t, "T", i.Output.String(), "accepted rune in input")
// Fork // Fork
child := i.Fork() child := i.Fork()
@ -21,14 +21,14 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
i.AcceptRune(r) // e i.AcceptRune(r) // e
r, _, _ = i.PeekRune(0) r, _, _ = i.PeekRune(0)
i.AcceptRune(r) // s i.AcceptRune(r) // s
AssertEqual(t, "es", i.String(), "result runes in fork") AssertEqual(t, "es", i.Output.String(), "result runes in fork")
AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset") AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset")
AssertEqual(t, 3, i.stackFrame.offset, "child offset") AssertEqual(t, 3, i.stackFrame.offset, "child offset")
// Merge fork back into parent // Merge fork back into parent
i.Merge(child) i.Merge(child)
i.Dispose(child) i.Dispose(child)
AssertEqual(t, "Tes", i.String(), "result runes in parent Input after Merge()") AssertEqual(t, "Tes", i.Output.String(), "result runes in parent Input after Merge()")
AssertEqual(t, 3, i.stackFrame.offset, "parent offset") AssertEqual(t, 3, i.stackFrame.offset, "parent offset")
} }
@ -44,17 +44,17 @@ func TestGivenForkedChildWhichAcceptedRune_AfterMerging_RuneEndsUpInParentResult
f2 := i.Fork() f2 := i.Fork()
r, _, _ = i.PeekRune(0) r, _, _ = i.PeekRune(0)
i.AcceptRune(r) // s i.AcceptRune(r) // s
AssertEqual(t, "s", i.String(), "f2 String()") AssertEqual(t, "s", i.Output.String(), "f2 String()")
AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A") AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A")
i.Merge(f2) i.Merge(f2)
i.Dispose(f2) i.Dispose(f2)
AssertEqual(t, "es", i.String(), "f1 String()") AssertEqual(t, "es", i.Output.String(), "f1 String()")
AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
i.Merge(f1) i.Merge(f1)
i.Dispose(f1) i.Dispose(f1)
AssertEqual(t, "Tes", i.String(), "top-level API String()") AssertEqual(t, "Tes", i.Output.String(), "top-level API String()")
AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A") AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
} }
@ -83,7 +83,7 @@ func TestFlushInput(t *testing.T) {
r, _, _ = i.PeekRune(0) r, _, _ = i.PeekRune(0)
i.AcceptRune(r) // o i.AcceptRune(r) // o
AssertEqual(t, "cool", i.String(), "end result") AssertEqual(t, "cool", i.Output.String(), "end result")
} }
func TestInputFlusherWrapper(t *testing.T) { func TestInputFlusherWrapper(t *testing.T) {
@ -92,19 +92,19 @@ func TestInputFlusherWrapper(t *testing.T) {
api := NewAPI("abaab") api := NewAPI("abaab")
runeA(api) runeA(api)
AssertEqual(t, 1, api.stackFrame.offset, "offset after 1 read") AssertEqual(t, 1, api.stackFrame.offset, "offset after 1 read")
AssertEqual(t, "a", api.String(), "runes after 1 read") AssertEqual(t, "a", api.Output.String(), "runes after 1 read")
flushB(api) flushB(api)
AssertEqual(t, 0, api.stackFrame.offset, "offset after 2 reads + input flush") AssertEqual(t, 0, api.stackFrame.offset, "offset after 2 reads + input flush")
AssertEqual(t, "ab", api.String(), "runes after 2 reads") AssertEqual(t, "ab", api.Output.String(), "runes after 2 reads")
runeA(api) runeA(api)
AssertEqual(t, 1, api.stackFrame.offset, "offset after 3 reads") AssertEqual(t, 1, api.stackFrame.offset, "offset after 3 reads")
AssertEqual(t, "aba", api.String(), "runes after 3 reads") AssertEqual(t, "aba", api.Output.String(), "runes after 3 reads")
runeA(api) runeA(api)
AssertEqual(t, 2, api.stackFrame.offset, "offset after 4 reads") AssertEqual(t, 2, api.stackFrame.offset, "offset after 4 reads")
AssertEqual(t, "abaa", api.String(), "runes after 4 reads") AssertEqual(t, "abaa", api.Output.String(), "runes after 4 reads")
flushB(api) flushB(api)
AssertEqual(t, 0, api.stackFrame.offset, "offset after 5 reads + input flush") AssertEqual(t, 0, api.stackFrame.offset, "offset after 5 reads + input flush")
AssertEqual(t, "abaab", api.String(), "runes after 5 reads") AssertEqual(t, "abaab", api.Output.String(), "runes after 5 reads")
} }
func AssertEqual(t *testing.T, expected interface{}, actual interface{}, forWhat string) { func AssertEqual(t *testing.T, expected interface{}, actual interface{}, forWhat string) {