Moving output functions to its own substruct of the API.
This commit is contained in:
parent
458d6f60a6
commit
9d98c9dff7
|
|
@ -29,8 +29,8 @@ type API struct {
|
|||
func (p *API) Peek(tokenHandler tokenize.Handler) bool {
|
||||
forkedAPI, ok := p.invokeHandler("Peek", tokenHandler)
|
||||
if ok {
|
||||
p.Result.Tokens = p.tokenAPI.Tokens()
|
||||
p.Result.Runes = p.tokenAPI.Runes()
|
||||
p.Result.Tokens = p.tokenAPI.Output.Tokens()
|
||||
p.Result.Runes = p.tokenAPI.Output.Runes()
|
||||
}
|
||||
p.tokenAPI.Dispose(forkedAPI)
|
||||
return ok
|
||||
|
|
@ -46,8 +46,8 @@ func (p *API) Accept(tokenHandler tokenize.Handler) bool {
|
|||
forkedAPI, ok := p.invokeHandler("Accept", tokenHandler)
|
||||
if ok {
|
||||
// Keep track of the results.
|
||||
p.Result.Tokens = p.tokenAPI.Tokens()
|
||||
p.Result.Runes = p.tokenAPI.Runes()
|
||||
p.Result.Tokens = p.tokenAPI.Output.Tokens()
|
||||
p.Result.Runes = p.tokenAPI.Output.Runes()
|
||||
|
||||
// Merge to the parent level.
|
||||
p.tokenAPI.Merge(forkedAPI)
|
||||
|
|
|
|||
113
tokenize/api.go
113
tokenize/api.go
|
|
@ -78,6 +78,7 @@ type API struct {
|
|||
stackFrames []stackFrame // the stack frames, containing stack level-specific data
|
||||
stackLevel int // the current stack level
|
||||
stackFrame *stackFrame // the current stack frame
|
||||
Output Output // provides output-related functionality
|
||||
}
|
||||
|
||||
type stackFrame struct {
|
||||
|
|
@ -93,6 +94,11 @@ type stackFrame struct {
|
|||
err error // can be used by a Handler to report a specific issue with the input
|
||||
}
|
||||
|
||||
// Output provides output-related functionality for the tokenize API.
|
||||
type Output struct {
|
||||
api *API
|
||||
}
|
||||
|
||||
const initialStackDepth = 64
|
||||
const initialTokenStoreLength = 64
|
||||
const initialByteStoreLength = 1024
|
||||
|
|
@ -107,6 +113,7 @@ func NewAPI(input interface{}) *API {
|
|||
tokens: make([]Token, initialTokenStoreLength),
|
||||
stackFrames: make([]stackFrame, initialStackDepth),
|
||||
}
|
||||
api.Output = Output{api: api}
|
||||
api.stackFrame = &api.stackFrames[0]
|
||||
|
||||
return api
|
||||
|
|
@ -460,73 +467,69 @@ func (i *API) FlushInput() bool {
|
|||
return false
|
||||
}
|
||||
|
||||
func (i *API) String() string {
|
||||
bytes := i.bytes[i.stackFrame.bytesStart:i.stackFrame.bytesEnd]
|
||||
func (o Output) String() string {
|
||||
bytes := o.api.bytes[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd]
|
||||
return string(bytes)
|
||||
}
|
||||
|
||||
func (i *API) Runes() []rune {
|
||||
bytes := i.bytes[i.stackFrame.bytesStart:i.stackFrame.bytesEnd]
|
||||
func (o Output) Runes() []rune {
|
||||
bytes := o.api.bytes[o.api.stackFrame.bytesStart:o.api.stackFrame.bytesEnd]
|
||||
return []rune(string(bytes))
|
||||
}
|
||||
|
||||
func (i *API) Rune(offset int) rune {
|
||||
r, _ := utf8.DecodeRune(i.bytes[i.stackFrame.bytesStart+offset:])
|
||||
func (o Output) Rune(offset int) rune {
|
||||
r, _ := utf8.DecodeRune(o.api.bytes[o.api.stackFrame.bytesStart+offset:])
|
||||
return r
|
||||
}
|
||||
|
||||
func (i *API) ClearBytes() {
|
||||
i.stackFrame.bytesEnd = i.stackFrame.bytesStart
|
||||
func (o Output) ClearData() {
|
||||
o.api.stackFrame.bytesEnd = o.api.stackFrame.bytesStart
|
||||
}
|
||||
|
||||
func (i *API) SetBytes(bytes ...byte) {
|
||||
i.ClearBytes()
|
||||
i.AddBytes(bytes...)
|
||||
func (o Output) SetBytes(bytes ...byte) {
|
||||
o.ClearData()
|
||||
o.AddBytes(bytes...)
|
||||
}
|
||||
|
||||
func (i *API) AddBytes(bytes ...byte) {
|
||||
func (o Output) AddBytes(bytes ...byte) {
|
||||
// Grow the runes capacity when needed.
|
||||
newBytesEnd := i.stackFrame.bytesEnd + len(bytes)
|
||||
if cap(i.bytes) < newBytesEnd {
|
||||
newBytesEnd := o.api.stackFrame.bytesEnd + len(bytes)
|
||||
if cap(o.api.bytes) < newBytesEnd {
|
||||
newBytes := make([]byte, newBytesEnd*2)
|
||||
copy(newBytes, i.bytes)
|
||||
i.bytes = newBytes
|
||||
copy(newBytes, o.api.bytes)
|
||||
o.api.bytes = newBytes
|
||||
}
|
||||
|
||||
copy(i.bytes[i.stackFrame.bytesEnd:], bytes)
|
||||
i.stackFrame.bytesEnd = newBytesEnd
|
||||
copy(o.api.bytes[o.api.stackFrame.bytesEnd:], bytes)
|
||||
o.api.stackFrame.bytesEnd = newBytesEnd
|
||||
}
|
||||
|
||||
func (i *API) ClearRunes() {
|
||||
i.stackFrame.bytesEnd = i.stackFrame.bytesStart
|
||||
func (o Output) SetRunes(runes ...rune) {
|
||||
o.ClearData()
|
||||
o.AddRunes(runes...)
|
||||
}
|
||||
|
||||
func (i *API) SetRunes(runes ...rune) {
|
||||
i.ClearRunes()
|
||||
i.AddRunes(runes...)
|
||||
}
|
||||
|
||||
func (i *API) AddRunes(runes ...rune) {
|
||||
func (o Output) AddRunes(runes ...rune) {
|
||||
// Grow the runes capacity when needed.
|
||||
runesAsString := string(runes)
|
||||
newBytesEnd := i.stackFrame.bytesEnd + len(runesAsString)
|
||||
if cap(i.bytes) < newBytesEnd {
|
||||
newBytesEnd := o.api.stackFrame.bytesEnd + len(runesAsString)
|
||||
if cap(o.api.bytes) < newBytesEnd {
|
||||
newBytes := make([]byte, newBytesEnd*2)
|
||||
copy(newBytes, i.bytes)
|
||||
i.bytes = newBytes
|
||||
copy(newBytes, o.api.bytes)
|
||||
o.api.bytes = newBytes
|
||||
}
|
||||
|
||||
copy(i.bytes[i.stackFrame.bytesEnd:], runesAsString)
|
||||
i.stackFrame.bytesEnd = newBytesEnd
|
||||
copy(o.api.bytes[o.api.stackFrame.bytesEnd:], runesAsString)
|
||||
o.api.stackFrame.bytesEnd = newBytesEnd
|
||||
}
|
||||
|
||||
func (i *API) AddString(s string) {
|
||||
i.AddBytes([]byte(s)...)
|
||||
func (o Output) AddString(s string) {
|
||||
o.AddBytes([]byte(s)...)
|
||||
}
|
||||
|
||||
func (i *API) SetString(s string) {
|
||||
i.ClearBytes()
|
||||
i.SetBytes([]byte(s)...)
|
||||
func (o Output) SetString(s string) {
|
||||
o.ClearData()
|
||||
o.SetBytes([]byte(s)...)
|
||||
}
|
||||
|
||||
func (i *API) Cursor() string {
|
||||
|
|
@ -536,38 +539,38 @@ func (i *API) Cursor() string {
|
|||
return fmt.Sprintf("line %d, column %d", i.stackFrame.line+1, i.stackFrame.column+1)
|
||||
}
|
||||
|
||||
func (i *API) Tokens() []Token {
|
||||
return i.tokens[i.stackFrame.tokenStart:i.stackFrame.tokenEnd]
|
||||
func (o Output) Tokens() []Token {
|
||||
return o.api.tokens[o.api.stackFrame.tokenStart:o.api.stackFrame.tokenEnd]
|
||||
}
|
||||
|
||||
func (i *API) Token(offset int) Token {
|
||||
return i.tokens[i.stackFrame.tokenStart+offset]
|
||||
func (o Output) Token(offset int) Token {
|
||||
return o.api.tokens[o.api.stackFrame.tokenStart+offset]
|
||||
}
|
||||
|
||||
func (i *API) TokenValue(offset int) interface{} {
|
||||
return i.tokens[i.stackFrame.tokenStart+offset].Value
|
||||
func (o Output) TokenValue(offset int) interface{} {
|
||||
return o.api.tokens[o.api.stackFrame.tokenStart+offset].Value
|
||||
}
|
||||
|
||||
func (i *API) ClearTokens() {
|
||||
i.stackFrame.tokenEnd = i.stackFrame.tokenStart
|
||||
func (o Output) ClearTokens() {
|
||||
o.api.stackFrame.tokenEnd = o.api.stackFrame.tokenStart
|
||||
}
|
||||
|
||||
func (i *API) SetTokens(tokens ...Token) {
|
||||
i.ClearTokens()
|
||||
i.AddTokens(tokens...)
|
||||
func (o Output) SetTokens(tokens ...Token) {
|
||||
o.ClearTokens()
|
||||
o.AddTokens(tokens...)
|
||||
}
|
||||
|
||||
func (i *API) AddTokens(tokens ...Token) {
|
||||
func (o Output) AddTokens(tokens ...Token) {
|
||||
// Grow the tokens capacity when needed.
|
||||
newTokenEnd := i.stackFrame.tokenEnd + len(tokens)
|
||||
if cap(i.tokens) < newTokenEnd {
|
||||
newTokenEnd := o.api.stackFrame.tokenEnd + len(tokens)
|
||||
if cap(o.api.tokens) < newTokenEnd {
|
||||
newTokens := make([]Token, newTokenEnd*2)
|
||||
copy(newTokens, i.tokens)
|
||||
i.tokens = newTokens
|
||||
copy(newTokens, o.api.tokens)
|
||||
o.api.tokens = newTokens
|
||||
}
|
||||
|
||||
for offset, t := range tokens {
|
||||
i.tokens[i.stackFrame.tokenEnd+offset] = t
|
||||
o.api.tokens[o.api.stackFrame.tokenEnd+offset] = t
|
||||
}
|
||||
i.stackFrame.tokenEnd = newTokenEnd
|
||||
o.api.stackFrame.tokenEnd = newTokenEnd
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ func ExampleNewAPI() {
|
|||
// r, err := api.NextRune()
|
||||
// fmt.Printf("Rune read from input; %c\n", r)
|
||||
// fmt.Printf("The error: %v\n", err)
|
||||
// fmt.Printf("API results: %q\n", api.String())
|
||||
// fmt.Printf("API results: %q\n", api.Output.String())
|
||||
|
||||
// // Output:
|
||||
// // Rune read from input; T
|
||||
|
|
@ -55,7 +55,7 @@ func ExampleAPI_AcceptRune() {
|
|||
// Reads 'e', but does not accept it to the API results.
|
||||
r, _, _ = api.PeekRune(0)
|
||||
|
||||
fmt.Printf("API results: %q\n", api.String())
|
||||
fmt.Printf("API results: %q\n", api.Output.String())
|
||||
|
||||
// Output:
|
||||
// API results: "Th"
|
||||
|
|
@ -74,7 +74,7 @@ func ExampleAPI_AcceptRunes() {
|
|||
// Accepts only 'T' and 'h' into the API results.
|
||||
api.AcceptRunes(r0, r1)
|
||||
|
||||
fmt.Printf("API results: %q\n", api.String())
|
||||
fmt.Printf("API results: %q\n", api.Output.String())
|
||||
|
||||
// Output:
|
||||
// API results: "Th"
|
||||
|
|
@ -99,7 +99,7 @@ func ExampleAPI_SkipRune() {
|
|||
}
|
||||
}
|
||||
|
||||
fmt.Printf("API results: %q\n", api.String())
|
||||
fmt.Printf("API results: %q\n", api.Output.String())
|
||||
|
||||
// Output:
|
||||
// API results: "eiuaeAIiae"
|
||||
|
|
@ -108,29 +108,29 @@ func ExampleAPI_SkipRune() {
|
|||
func ExampleAPI_modifyingResults() {
|
||||
api := tokenize.NewAPI("")
|
||||
|
||||
api.AddString("Some runes")
|
||||
api.AddRunes(' ', 'a', 'd', 'd', 'e', 'd')
|
||||
api.AddRunes(' ', 'i', 'n', ' ')
|
||||
api.AddString("various ways")
|
||||
fmt.Printf("API result first 10 runes: %q\n", api.Runes()[0:10])
|
||||
fmt.Printf("API result runes as string: %q\n", api.String())
|
||||
api.Output.AddString("Some runes")
|
||||
api.Output.AddRunes(' ', 'a', 'd', 'd', 'e', 'd')
|
||||
api.Output.AddRunes(' ', 'i', 'n', ' ')
|
||||
api.Output.AddString("various ways")
|
||||
fmt.Printf("API result first 10 runes: %q\n", api.Output.Runes()[0:10])
|
||||
fmt.Printf("API result runes as string: %q\n", api.Output.String())
|
||||
|
||||
api.SetString("new ")
|
||||
api.AddString("set ")
|
||||
api.AddString("of ")
|
||||
api.AddRunes('r', 'u', 'n', 'e', 's')
|
||||
fmt.Printf("API result runes as string: %q\n", api.String())
|
||||
fmt.Printf("API result runes: %q\n", api.Runes())
|
||||
fmt.Printf("API third rune: %q\n", api.Rune(2))
|
||||
api.Output.SetString("new ")
|
||||
api.Output.AddString("set ")
|
||||
api.Output.AddString("of ")
|
||||
api.Output.AddRunes('r', 'u', 'n', 'e', 's')
|
||||
fmt.Printf("API result runes as string: %q\n", api.Output.String())
|
||||
fmt.Printf("API result runes: %q\n", api.Output.Runes())
|
||||
fmt.Printf("API third rune: %q\n", api.Output.Rune(2))
|
||||
|
||||
api.AddTokens(tokenize.Token{
|
||||
api.Output.AddTokens(tokenize.Token{
|
||||
Type: 42,
|
||||
Value: "towel"})
|
||||
api.AddTokens(tokenize.Token{
|
||||
api.Output.AddTokens(tokenize.Token{
|
||||
Type: 73,
|
||||
Value: "Zaphod"})
|
||||
fmt.Printf("API result tokens: %v\n", api.Tokens())
|
||||
fmt.Printf("API second result token: %v\n", api.Token(1))
|
||||
fmt.Printf("API result tokens: %v\n", api.Output.Tokens())
|
||||
fmt.Printf("API second result token: %v\n", api.Output.Token(1))
|
||||
|
||||
// Output:
|
||||
// API result first 10 runes: ['S' 'o' 'm' 'e' ' ' 'r' 'u' 'n' 'e' 's']
|
||||
|
|
@ -149,18 +149,18 @@ func ExampleAPI_Reset() {
|
|||
api.AcceptRune(r)
|
||||
r, _, _ = api.PeekRune(0) // read 'e'
|
||||
api.AcceptRune(r)
|
||||
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor())
|
||||
fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor())
|
||||
|
||||
// Reset clears the results.
|
||||
api.Reset()
|
||||
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor())
|
||||
fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor())
|
||||
|
||||
// So then doing the same read operations, the same data are read.
|
||||
r, _, _ = api.PeekRune(0) // read 'V'
|
||||
api.AcceptRune(r)
|
||||
r, _, _ = api.PeekRune(0) // read 'e'
|
||||
api.AcceptRune(r)
|
||||
fmt.Printf("API results: %q at %s\n", api.String(), api.Cursor())
|
||||
fmt.Printf("API results: %q at %s\n", api.Output.String(), api.Cursor())
|
||||
|
||||
// Output:
|
||||
// API results: "Ve" at line 1, column 3
|
||||
|
|
@ -246,23 +246,23 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
|
|||
r, _, _ := api.PeekRune(0)
|
||||
AssertEqual(t, 'a', r, "child4 rune 1")
|
||||
api.AcceptRune(r)
|
||||
AssertEqual(t, "a", api.String(), "child4 runes after rune 1")
|
||||
AssertEqual(t, "a", api.Output.String(), "child4 runes after rune 1")
|
||||
|
||||
// Read another rune 'b' from child4.
|
||||
r, _, _ = api.PeekRune(0)
|
||||
AssertEqual(t, 'b', r, "child4 rune 2")
|
||||
api.AcceptRune(r)
|
||||
AssertEqual(t, "ab", api.String(), "child4 runes after rune 2")
|
||||
AssertEqual(t, "ab", api.Output.String(), "child4 runes after rune 2")
|
||||
|
||||
// Merge "ab" from child4 to child3.
|
||||
api.Merge(child4)
|
||||
AssertEqual(t, "", api.String(), "child4 runes after first merge")
|
||||
AssertEqual(t, "", api.Output.String(), "child4 runes after first merge")
|
||||
|
||||
// Read some more from child4.
|
||||
r, _, _ = api.PeekRune(0)
|
||||
AssertEqual(t, 'c', r, "child4 rune 3")
|
||||
api.AcceptRune(r)
|
||||
AssertEqual(t, "c", api.String(), "child4 runes after rune 1")
|
||||
AssertEqual(t, "c", api.Output.String(), "child4 runes after rune 1")
|
||||
AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child4 rune 3")
|
||||
|
||||
// Merge "c" from child4 to child3.
|
||||
|
|
@ -272,7 +272,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
|
|||
api.Dispose(child4)
|
||||
|
||||
// Child3 should now have the compbined results "abc" from child4's work.
|
||||
AssertEqual(t, "abc", api.String(), "child3 after merge of child4")
|
||||
AssertEqual(t, "abc", api.Output.String(), "child3 after merge of child4")
|
||||
AssertEqual(t, "line 1, column 4", api.Cursor(), "cursor child3 rune 3, after merge of child4")
|
||||
|
||||
// Now read some data from child3.
|
||||
|
|
@ -288,7 +288,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
|
|||
AssertEqual(t, 'f', r, "child3 rune 5")
|
||||
api.AcceptRune(r)
|
||||
|
||||
AssertEqual(t, "abcdef", api.String(), "child3 total result after rune 6")
|
||||
AssertEqual(t, "abcdef", api.Output.String(), "child3 total result after rune 6")
|
||||
|
||||
// Temporarily go some new forks from here, but don't use their outcome.
|
||||
child3sub1 := api.Fork()
|
||||
|
|
@ -308,7 +308,7 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
|
|||
api.Merge(child3)
|
||||
api.Dispose(child3)
|
||||
|
||||
AssertEqual(t, "abcdef", api.String(), "child2 total result after merge of child3")
|
||||
AssertEqual(t, "abcdef", api.Output.String(), "child2 total result after merge of child3")
|
||||
AssertEqual(t, "line 1, column 7", api.Cursor(), "cursor child2 after merge child3")
|
||||
|
||||
// Merge child2 to child1 and dispose of it.
|
||||
|
|
@ -328,23 +328,23 @@ func TestMultipleLevelsOfForksAndMerges(t *testing.T) {
|
|||
r, _, _ = api.PeekRune(0)
|
||||
api.AcceptRune(r)
|
||||
|
||||
AssertEqual(t, "abcdefg", api.String(), "api string end result")
|
||||
AssertEqual(t, "abcdefg", api.Output.String(), "api string end result")
|
||||
AssertEqual(t, "line 1, column 8", api.Cursor(), "api cursor end result")
|
||||
}
|
||||
|
||||
func TestClearRunes(t *testing.T) {
|
||||
func TestClearData(t *testing.T) {
|
||||
api := tokenize.NewAPI("Laphroaig")
|
||||
r, _, _ := api.PeekRune(0) // Read 'L'
|
||||
api.AcceptRune(r) // Add to runes
|
||||
r, _, _ = api.PeekRune(0) // Read 'a'
|
||||
api.AcceptRune(r) // Add to runes
|
||||
api.ClearRunes() // Clear the runes, giving us a fresh start.
|
||||
api.Output.ClearData() // Clear the runes, giving us a fresh start.
|
||||
r, _, _ = api.PeekRune(0) // Read 'p'
|
||||
api.AcceptRune(r) // Add to runes
|
||||
r, _, _ = api.PeekRune(0) // Read 'r'
|
||||
api.AcceptRune(r) // Add to runes
|
||||
|
||||
AssertEqual(t, "ph", api.String(), "api string end result")
|
||||
AssertEqual(t, "ph", api.Output.String(), "api string end result")
|
||||
}
|
||||
|
||||
func TestMergeScenariosForTokens(t *testing.T) {
|
||||
|
|
@ -355,34 +355,34 @@ func TestMergeScenariosForTokens(t *testing.T) {
|
|||
token3 := tokenize.Token{Value: 3}
|
||||
token4 := tokenize.Token{Value: 4}
|
||||
|
||||
api.SetTokens(token1)
|
||||
tokens := api.Tokens()
|
||||
api.Output.SetTokens(token1)
|
||||
tokens := api.Output.Tokens()
|
||||
AssertEqual(t, 1, len(tokens), "Tokens 1")
|
||||
|
||||
child := api.Fork()
|
||||
|
||||
tokens = api.Tokens()
|
||||
tokens = api.Output.Tokens()
|
||||
AssertEqual(t, 0, len(tokens), "Tokens 2")
|
||||
|
||||
api.AddTokens(token2)
|
||||
api.Output.AddTokens(token2)
|
||||
|
||||
api.Merge(child)
|
||||
api.Dispose(child)
|
||||
|
||||
tokens = api.Tokens()
|
||||
tokens = api.Output.Tokens()
|
||||
AssertEqual(t, 2, len(tokens), "Tokens 3")
|
||||
|
||||
child = api.Fork()
|
||||
api.AddTokens(token3)
|
||||
api.Output.AddTokens(token3)
|
||||
api.Reset()
|
||||
api.AddTokens(token4)
|
||||
api.Output.AddTokens(token4)
|
||||
|
||||
api.Merge(child)
|
||||
api.Dispose(child)
|
||||
|
||||
tokens = api.Tokens()
|
||||
tokens = api.Output.Tokens()
|
||||
AssertEqual(t, 3, len(tokens), "Tokens 4")
|
||||
AssertEqual(t, 1, api.TokenValue(0).(int), "Tokens 4, value 0")
|
||||
AssertEqual(t, 2, api.TokenValue(1).(int), "Tokens 4, value 1")
|
||||
AssertEqual(t, 4, api.TokenValue(2).(int), "Tokens 4, value 2")
|
||||
AssertEqual(t, 1, api.Output.TokenValue(0).(int), "Tokens 4, value 0")
|
||||
AssertEqual(t, 2, api.Output.TokenValue(1).(int), "Tokens 4, value 1")
|
||||
AssertEqual(t, 4, api.Output.TokenValue(2).(int), "Tokens 4, value 2")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -987,7 +987,7 @@ func MatchIntegerBetween(min int64, max int64) Handler {
|
|||
if !digits(t) {
|
||||
return false
|
||||
}
|
||||
value, _ := strconv.ParseInt(t.String(), 10, 64)
|
||||
value, _ := strconv.ParseInt(t.Output.String(), 10, 64)
|
||||
if value < min || value > max {
|
||||
return false
|
||||
}
|
||||
|
|
@ -1346,16 +1346,16 @@ func MatchOctet(normalize bool) Handler {
|
|||
if !max3Digits(t) {
|
||||
return false
|
||||
}
|
||||
value, _ := strconv.ParseInt(t.String(), 10, 16)
|
||||
value, _ := strconv.ParseInt(t.Output.String(), 10, 16)
|
||||
if value > 255 {
|
||||
return false
|
||||
}
|
||||
if normalize {
|
||||
runes := t.Runes()
|
||||
runes := t.Output.Runes()
|
||||
for len(runes) > 1 && runes[0] == '0' {
|
||||
runes = runes[1:]
|
||||
}
|
||||
t.SetRunes(runes...)
|
||||
t.Output.SetRunes(runes...)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -1397,13 +1397,13 @@ func MatchIPv4Netmask(normalize bool) Handler {
|
|||
}
|
||||
|
||||
// Check if the mask is provided in canonical form (at the binary level, ones followed by zeroes).
|
||||
mask := net.IPv4Mask(t.TokenValue(0).(byte), t.TokenValue(1).(byte), t.TokenValue(2).(byte), t.TokenValue(3).(byte))
|
||||
mask := net.IPv4Mask(t.Output.TokenValue(0).(byte), t.Output.TokenValue(1).(byte), t.Output.TokenValue(2).(byte), t.Output.TokenValue(3).(byte))
|
||||
ones, bits := mask.Size()
|
||||
if ones == 0 && bits == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
t.ClearTokens()
|
||||
t.Output.ClearTokens()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
@ -1432,18 +1432,18 @@ func MatchIPv4Net(normalize bool) Handler {
|
|||
return true
|
||||
}
|
||||
|
||||
maskToken := t.Token(1)
|
||||
maskToken := t.Output.Token(1)
|
||||
if maskToken.Type == "cidr" {
|
||||
t.SetString(fmt.Sprintf("%s/%d", t.TokenValue(0), t.TokenValue(1).(uint8)))
|
||||
t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), t.Output.TokenValue(1).(uint8)))
|
||||
} else {
|
||||
o := strings.Split(t.TokenValue(1).(string), ".")
|
||||
o := strings.Split(t.Output.TokenValue(1).(string), ".")
|
||||
b := func(idx int) byte { i, _ := strconv.Atoi(o[idx]); return byte(i) }
|
||||
mask := net.IPv4Mask(b(0), b(1), b(2), b(3))
|
||||
bits, _ := mask.Size()
|
||||
t.SetString(fmt.Sprintf("%s/%d", t.TokenValue(0), bits))
|
||||
t.Output.SetString(fmt.Sprintf("%s/%d", t.Output.TokenValue(0), bits))
|
||||
}
|
||||
|
||||
t.ClearTokens()
|
||||
t.Output.ClearTokens()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
@ -1472,13 +1472,13 @@ func MatchIPv6(normalize bool) Handler {
|
|||
}
|
||||
|
||||
// Invalid IPv6, when net.ParseIP() cannot handle it.
|
||||
parsed := net.ParseIP(t.String())
|
||||
parsed := net.ParseIP(t.Output.String())
|
||||
if parsed == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if normalize {
|
||||
t.SetString(parsed.String())
|
||||
t.Output.SetString(parsed.String())
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -1501,8 +1501,8 @@ func matchCIDRMask(bits int64, normalize bool) Handler {
|
|||
if !mask(t) {
|
||||
return false
|
||||
}
|
||||
bits, _ := strconv.Atoi(t.String())
|
||||
t.SetString(fmt.Sprintf("%d", bits))
|
||||
bits, _ := strconv.Atoi(t.Output.String())
|
||||
t.Output.SetString(fmt.Sprintf("%d", bits))
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
@ -1643,8 +1643,8 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler {
|
|||
return func(t *API) bool {
|
||||
child := t.Fork()
|
||||
if handler(t) {
|
||||
s := modfunc(t.String())
|
||||
t.SetString(s)
|
||||
s := modfunc(t.Output.String())
|
||||
t.Output.SetString(s)
|
||||
t.Merge(child)
|
||||
t.Dispose(child)
|
||||
return true
|
||||
|
|
@ -1661,7 +1661,7 @@ func ModifyByCallback(handler Handler, modfunc func(string) string) Handler {
|
|||
// an 'n'-character).
|
||||
func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler {
|
||||
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
|
||||
literal := t.String()
|
||||
literal := t.Output.String()
|
||||
return literal
|
||||
})
|
||||
}
|
||||
|
|
@ -1673,7 +1673,7 @@ func MakeStrLiteralToken(toktype interface{}, handler Handler) Handler {
|
|||
func MakeStrInterpretedToken(toktype interface{}, handler Handler) Handler {
|
||||
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
|
||||
// TODO ERROR HANDLING
|
||||
interpreted, _ := interpretString(t.String())
|
||||
interpreted, _ := interpretString(t.Output.String())
|
||||
return interpreted
|
||||
})
|
||||
}
|
||||
|
|
@ -1697,7 +1697,7 @@ func interpretString(str string) (string, error) {
|
|||
func MakeRuneToken(toktype interface{}, handler Handler) Handler {
|
||||
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
|
||||
// TODO ERROR HANDLING --- not a 1 rune input
|
||||
return t.Rune(0)
|
||||
return t.Output.Rune(0)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1707,7 +1707,7 @@ func MakeRuneToken(toktype interface{}, handler Handler) Handler {
|
|||
func MakeByteToken(toktype interface{}, handler Handler) Handler {
|
||||
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
|
||||
// TODO ERROR HANDLING --- not a 1 byte input
|
||||
return byte(t.Rune(0))
|
||||
return byte(t.Output.Rune(0))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1912,7 +1912,7 @@ func MakeBooleanToken(toktype interface{}, handler Handler) Handler {
|
|||
|
||||
func makeStrconvToken(name string, toktype interface{}, handler Handler, convert func(s string) (interface{}, error)) Handler {
|
||||
return MakeTokenByCallback(toktype, handler, func(t *API) interface{} {
|
||||
value, err := convert(t.String())
|
||||
value, err := convert(t.Output.String())
|
||||
if err != nil {
|
||||
// TODO meh, panic feels so bad here. Maybe just turn this case into "no match"?
|
||||
panic(fmt.Sprintf("%s token invalid (%s)", name, err))
|
||||
|
|
@ -1943,7 +1943,7 @@ func MakeTokenByCallback(toktype interface{}, handler Handler, makeValue func(t
|
|||
// tokens will end up in the order "date", "year", "month", "day". When we'd add the
|
||||
// token to the child here, the order would have been "year", "month", "day", "date".
|
||||
token := Token{Type: toktype, Value: makeValue(t)}
|
||||
t.AddTokens(token)
|
||||
t.Output.AddTokens(token)
|
||||
t.Merge(child)
|
||||
t.Dispose(child)
|
||||
|
||||
|
|
@ -1960,10 +1960,10 @@ func MakeTokenGroup(toktype interface{}, handler Handler) Handler {
|
|||
return func(t *API) bool {
|
||||
child := t.Fork()
|
||||
if handler(t) {
|
||||
tokens := t.Tokens()
|
||||
tokens := t.Output.Tokens()
|
||||
tokensCopy := make([]Token, len(tokens))
|
||||
copy(tokensCopy, tokens)
|
||||
t.SetTokens(Token{Type: toktype, Value: tokensCopy})
|
||||
t.Output.SetTokens(Token{Type: toktype, Value: tokensCopy})
|
||||
t.Merge(child)
|
||||
t.Dispose(child)
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -47,8 +47,8 @@ func New(tokenHandler Handler) Func {
|
|||
return nil, err
|
||||
}
|
||||
result := &Result{
|
||||
Runes: api.Runes(),
|
||||
Tokens: api.Tokens(),
|
||||
Runes: api.Output.Runes(),
|
||||
Tokens: api.Output.Tokens(),
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ func TestInputCanAcceptRunesFromReader(t *testing.T) {
|
|||
r2, _, _ := i.PeekRune(1)
|
||||
i.AcceptRunes(r1, r2)
|
||||
|
||||
AssertEqual(t, "Tes", i.String(), "i.String()")
|
||||
AssertEqual(t, "Tes", i.Output.String(), "i.String()")
|
||||
}
|
||||
|
||||
func TestCallingMergeOnTopLevelAPI_Panics(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
|
|||
i := NewAPI("Testing")
|
||||
r, _, _ := i.PeekRune(0)
|
||||
i.AcceptRune(r) // T
|
||||
AssertEqual(t, "T", i.String(), "accepted rune in input")
|
||||
AssertEqual(t, "T", i.Output.String(), "accepted rune in input")
|
||||
|
||||
// Fork
|
||||
child := i.Fork()
|
||||
|
|
@ -21,14 +21,14 @@ func TestFork_CreatesForkOfInputAtSameCursorPosition(t *testing.T) {
|
|||
i.AcceptRune(r) // e
|
||||
r, _, _ = i.PeekRune(0)
|
||||
i.AcceptRune(r) // s
|
||||
AssertEqual(t, "es", i.String(), "result runes in fork")
|
||||
AssertEqual(t, "es", i.Output.String(), "result runes in fork")
|
||||
AssertEqual(t, 1, i.stackFrames[i.stackLevel-1].offset, "parent offset")
|
||||
AssertEqual(t, 3, i.stackFrame.offset, "child offset")
|
||||
|
||||
// Merge fork back into parent
|
||||
i.Merge(child)
|
||||
i.Dispose(child)
|
||||
AssertEqual(t, "Tes", i.String(), "result runes in parent Input after Merge()")
|
||||
AssertEqual(t, "Tes", i.Output.String(), "result runes in parent Input after Merge()")
|
||||
AssertEqual(t, 3, i.stackFrame.offset, "parent offset")
|
||||
}
|
||||
|
||||
|
|
@ -44,17 +44,17 @@ func TestGivenForkedChildWhichAcceptedRune_AfterMerging_RuneEndsUpInParentResult
|
|||
f2 := i.Fork()
|
||||
r, _, _ = i.PeekRune(0)
|
||||
i.AcceptRune(r) // s
|
||||
AssertEqual(t, "s", i.String(), "f2 String()")
|
||||
AssertEqual(t, "s", i.Output.String(), "f2 String()")
|
||||
AssertEqual(t, 3, i.stackFrame.offset, "f2.offset A")
|
||||
|
||||
i.Merge(f2)
|
||||
i.Dispose(f2)
|
||||
AssertEqual(t, "es", i.String(), "f1 String()")
|
||||
AssertEqual(t, "es", i.Output.String(), "f1 String()")
|
||||
AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
|
||||
|
||||
i.Merge(f1)
|
||||
i.Dispose(f1)
|
||||
AssertEqual(t, "Tes", i.String(), "top-level API String()")
|
||||
AssertEqual(t, "Tes", i.Output.String(), "top-level API String()")
|
||||
AssertEqual(t, 3, i.stackFrame.offset, "f1.offset A")
|
||||
}
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ func TestFlushInput(t *testing.T) {
|
|||
r, _, _ = i.PeekRune(0)
|
||||
i.AcceptRune(r) // o
|
||||
|
||||
AssertEqual(t, "cool", i.String(), "end result")
|
||||
AssertEqual(t, "cool", i.Output.String(), "end result")
|
||||
}
|
||||
|
||||
func TestInputFlusherWrapper(t *testing.T) {
|
||||
|
|
@ -92,19 +92,19 @@ func TestInputFlusherWrapper(t *testing.T) {
|
|||
api := NewAPI("abaab")
|
||||
runeA(api)
|
||||
AssertEqual(t, 1, api.stackFrame.offset, "offset after 1 read")
|
||||
AssertEqual(t, "a", api.String(), "runes after 1 read")
|
||||
AssertEqual(t, "a", api.Output.String(), "runes after 1 read")
|
||||
flushB(api)
|
||||
AssertEqual(t, 0, api.stackFrame.offset, "offset after 2 reads + input flush")
|
||||
AssertEqual(t, "ab", api.String(), "runes after 2 reads")
|
||||
AssertEqual(t, "ab", api.Output.String(), "runes after 2 reads")
|
||||
runeA(api)
|
||||
AssertEqual(t, 1, api.stackFrame.offset, "offset after 3 reads")
|
||||
AssertEqual(t, "aba", api.String(), "runes after 3 reads")
|
||||
AssertEqual(t, "aba", api.Output.String(), "runes after 3 reads")
|
||||
runeA(api)
|
||||
AssertEqual(t, 2, api.stackFrame.offset, "offset after 4 reads")
|
||||
AssertEqual(t, "abaa", api.String(), "runes after 4 reads")
|
||||
AssertEqual(t, "abaa", api.Output.String(), "runes after 4 reads")
|
||||
flushB(api)
|
||||
AssertEqual(t, 0, api.stackFrame.offset, "offset after 5 reads + input flush")
|
||||
AssertEqual(t, "abaab", api.String(), "runes after 5 reads")
|
||||
AssertEqual(t, "abaab", api.Output.String(), "runes after 5 reads")
|
||||
}
|
||||
|
||||
func AssertEqual(t *testing.T, expected interface{}, actual interface{}, forWhat string) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue