package parsekit_test import ( "testing" "git.makaay.nl/mauricem/go-parsekit" ) func TestWithinTokenHandler_AcceptIncludesRuneInOutput(t *testing.T) { parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool { for i := 0; i < 20; i++ { t.NextRune() t.Accept() } return true }, "test") result, _ := parser.Execute("This is some random data to parse") if result.String() != "This is some random " { t.Fatalf("Got unexpected output from TokenHandler: %s", result.String()) } } func TestWithinTokenHandler_TokensCanBeEmitted(t *testing.T) { parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool { t.Result().AddToken(&parsekit.Token{ Type: "PI", Runes: []rune("π"), Value: 3.1415, }) t.Result().AddToken(&parsekit.Token{ Type: nil, Runes: []rune("yes"), Value: true, }) return true }, "test") result, _ := parser.Execute("doesn't matter") if len(result.Tokens()) != 2 { t.Fatalf("Wrong number of tokens in result, expected 2, got %d", len(result.Tokens())) } if result.Token(0).Value != 3.1415 { t.Fatal("Token 0 value not 3.1415") } if string(result.Token(0).Runes) != "π" { t.Fatal("Token 0 runes not \"π\"") } if result.Token(0).Type != "PI" { t.Fatal("Token 0 type not \"PI\"") } if result.Token(1).Value != true { t.Fatal("Token 1 value not true") } if string(result.Token(1).Runes) != "yes" { t.Fatal("Token 1 runes not \"yes\"") } if result.Token(1).Type != nil { t.Fatal("Token 1 type not nil") } } func TestUsingTokenParserCombinators_TokensCanBeEmitted(t *testing.T) { var tok, c, a = parsekit.T, parsekit.C, parsekit.A fooToken := tok.Str("ASCII", c.OneOrMore(a.ASCII)) parser := parsekit.NewTokenizer(fooToken, "something") input := "This is fine ASCII Åltho hère öt endĩt!" result, err := parser.Execute(input) if err != nil { t.Fatalf("Unexpected error from parser: %s", err) } if result.String() != "This is fine ASCII " { t.Fatalf("result.String() contains unexpected data: %s", result.String()) } } func TestUsingTokenParserCombinators_TokensCanBeNested(t *testing.T) { var c, m, tok, a = parsekit.C, parsekit.M, parsekit.T, parsekit.A fooToken := c.Seq( m.Drop(c.ZeroOrMore(a.Asterisk)), tok.Str("COMBI", c.Seq( tok.Str("ASCII", m.TrimSpace(c.OneOrMore(a.ASCII))), tok.Str("UTF8", m.TrimSpace(c.OneOrMore(c.Except(a.Asterisk, a.AnyRune)))), )), m.Drop(c.ZeroOrMore(a.Asterisk)), ) parser := parsekit.NewTokenizer(fooToken, "something") input := "*** This is fine ASCII Åltho hère öt endĩt! ***" output := "This is fine ASCIIÅltho hère öt endĩt!" result, err := parser.Execute(input) if err != nil { t.Fatalf("Unexpected error from parser: %s", err) } if result.String() != output { t.Fatalf("result.String() contains unexpected data: %s", result.String()) } if result.Token(0).Type != "COMBI" { t.Fatalf("Token 0 has unexpected type: %s", result.Token(0).Type) } if result.Token(0).Value != "This is fine ASCIIÅltho hère öt endĩt!" { t.Fatalf("Token 0 has unexpected value: %s", result.Token(0).Value) } if result.Token(1).Value != "This is fine ASCII" { t.Fatalf("Token 1 has unexpected value: %s", result.Token(0).Value) } if result.Token(2).Value != "Åltho hère öt endĩt!" { t.Fatalf("Token 2 has unexpected value: %s", result.Token(0).Value) } } func TestGivenNextRuneNotCalled_CallToAcceptPanics(t *testing.T) { parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool { t.Accept() return false }, "test") parsekit.AssertPanic(t, parsekit.PanicT{ Function: func() { parser.Execute("input string") }, Regexp: true, Expect: `parsekit\.TokenAPI\.Accept\(\): Accept\(\) called at ` + `/.*/tokenhandler_test\.go:\d+ without first calling NextRune\(\)`}) } func TestGivenAcceptNotCalled_CallToNextRunePanics(t *testing.T) { parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool { t.NextRune() t.NextRune() return false }, "test") parsekit.AssertPanic(t, parsekit.PanicT{ Function: func() { parser.Execute("input string") }, Regexp: true, Expect: `parsekit\.TokenAPI\.NextRune\(\): NextRune\(\) called at ` + `/.*/tokenhandler_test\.go:\d+ without a prior call to Accept\(\)`}) } func TestGivenNextRuneReturningNotOk_CallToAcceptPanics(t *testing.T) { parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool { t.NextRune() t.Accept() return false }, "test") parsekit.AssertPanic(t, parsekit.PanicT{ Function: func() { parser.Execute("") }, Regexp: true, Expect: `parsekit\.TokenAPI\.Accept\(\): Accept\(\) called at ` + `/.*/tokenhandler_test.go:\d+, but the prior call to NextRune\(\) failed`}) } func TestGivenRootTokenAPI_CallingMergePanics(t *testing.T) { parsekit.AssertPanic(t, parsekit.PanicT{ Function: func() { a := parsekit.TokenAPI{} a.Merge() }, Regexp: true, Expect: `parsekit\.TokenAPI\.Merge\(\): Merge\(\) called at ` + `/.*/tokenhandler_test\.go:\d+ on a non-forked TokenAPI`}) }