go-parsekit/tokenhandler_test.go

package parsekit_test

import (
	"testing"

	"git.makaay.nl/mauricem/go-parsekit"
)

func TestWithinTokenHandler_AcceptIncludesRuneInOutput(t *testing.T) {
	parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool {
		for i := 0; i < 20; i++ {
			t.NextRune()
			t.Accept()
		}
		return true
	}, "test")
	result, _ := parser.Execute("This is some random data to parse")
	if result.String() != "This is some random " {
		t.Fatalf("Got unexpected output from TokenHandler: %s", result.String())
	}
}

func TestWithinTokenHandler_TokensCanBeEmitted(t *testing.T) {
	parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool {
		t.Result().AddToken(&parsekit.Token{
			Type:  "PI",
			Runes: []rune("π"),
			Value: 3.1415,
		})
		t.Result().AddToken(&parsekit.Token{
			Type:  nil,
			Runes: []rune("yes"),
			Value: true,
		})
		return true
	}, "test")
	result, _ := parser.Execute("doesn't matter")
	if len(result.Tokens()) != 2 {
		t.Fatalf("Wrong number of tokens in result, expected 2, got %d", len(result.Tokens()))
	}
	if result.Token(0).Value != 3.1415 {
		t.Fatal("Token 0 value not 3.1415")
	}
	if string(result.Token(0).Runes) != "π" {
		t.Fatal("Token 0 runes not \"π\"")
	}
	if result.Token(0).Type != "PI" {
		t.Fatal("Token 0 type not \"PI\"")
	}
	if result.Token(1).Value != true {
		t.Fatal("Token 1 value not true")
	}
	if string(result.Token(1).Runes) != "yes" {
		t.Fatal("Token 1 runes not \"yes\"")
	}
	if result.Token(1).Type != nil {
		t.Fatal("Token 1 type not nil")
	}
}

func TestUsingTokenParserCombinators_TokensCanBeEmitted(t *testing.T) {
	var tok, c, a = parsekit.T, parsekit.C, parsekit.A
	fooToken := tok.Str("ASCII", c.OneOrMore(a.ASCII))
	parser := parsekit.NewTokenizer(fooToken, "something")
	input := "This is fine ASCII Åltho hère öt endĩt!"
	result, err := parser.Execute(input)

	if err != nil {
		t.Fatalf("Unexpected error from parser: %s", err)
	}
	if result.String() != "This is fine ASCII " {
		t.Fatalf("result.String() contains unexpected data: %s", result.String())
	}
}

func TestUsingTokenParserCombinators_TokensCanBeNested(t *testing.T) {
	var c, m, tok, a = parsekit.C, parsekit.M, parsekit.T, parsekit.A
	fooToken := c.Seq(
		m.Drop(c.ZeroOrMore(a.Asterisk)),
		tok.Str("COMBI", c.Seq(
			tok.Str("ASCII", m.TrimSpace(c.OneOrMore(a.ASCII))),
			tok.Str("UTF8", m.TrimSpace(c.OneOrMore(c.Except(a.Asterisk, a.AnyRune)))),
		)),
		m.Drop(c.ZeroOrMore(a.Asterisk)),
	)
	parser := parsekit.NewTokenizer(fooToken, "something")
	input := "*** This is fine ASCII  Åltho hère öt endĩt! ***"
	output := "This is fine ASCIIÅltho hère öt endĩt!"
	result, err := parser.Execute(input)

	if err != nil {
		t.Fatalf("Unexpected error from parser: %s", err)
	}
	if result.String() != output {
		t.Fatalf("result.String() contains unexpected data: %s", result.String())
	}
	if result.Token(0).Type != "COMBI" {
		t.Fatalf("Token 0 has unexpected type: %s", result.Token(0).Type)
	}
	if result.Token(0).Value != "This is fine ASCIIÅltho hère öt endĩt!" {
		t.Fatalf("Token 0 has unexpected value: %s", result.Token(0).Value)
	}
	if result.Token(1).Value != "This is fine ASCII" {
		t.Fatalf("Token 1 has unexpected value: %s", result.Token(0).Value)
	}
	if result.Token(2).Value != "Åltho hère öt endĩt!" {
		t.Fatalf("Token 2 has unexpected value: %s", result.Token(0).Value)
	}
}

func TestGivenNextRuneNotCalled_CallToAcceptPanics(t *testing.T) {
	parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool {
		t.Accept()
		return false
	}, "test")
	parsekit.AssertPanic(t, parsekit.PanicT{
		Function: func() { parser.Execute("input string") },
		Regexp:   true,
		Expect: `parsekit\.TokenAPI\.Accept\(\): Accept\(\) called at ` +
			`/.*/tokenhandler_test\.go:\d+ without first calling NextRune\(\)`})
}

func TestGivenAcceptNotCalled_CallToNextRunePanics(t *testing.T) {
	parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool {
		t.NextRune()
		t.NextRune()
		return false
	}, "test")
	parsekit.AssertPanic(t, parsekit.PanicT{
		Function: func() { parser.Execute("input string") },
		Regexp:   true,
		Expect: `parsekit\.TokenAPI\.NextRune\(\): NextRune\(\) called at ` +
			`/.*/tokenhandler_test\.go:\d+ without a prior call to Accept\(\)`})
}

func TestGivenNextRuneReturningNotOk_CallToAcceptPanics(t *testing.T) {
	parser := parsekit.NewTokenizer(func(t *parsekit.TokenAPI) bool {
		t.NextRune()
		t.Accept()
		return false
	}, "test")
	parsekit.AssertPanic(t, parsekit.PanicT{
		Function: func() { parser.Execute("") },
		Regexp:   true,
		Expect: `parsekit\.TokenAPI\.Accept\(\): Accept\(\) called at ` +
			`/.*/tokenhandler_test.go:\d+, but the prior call to NextRune\(\) failed`})
}

func TestGivenRootTokenAPI_CallingMergePanics(t *testing.T) {
	parsekit.AssertPanic(t, parsekit.PanicT{
		Function: func() {
			a := parsekit.TokenAPI{}
			a.Merge()
		},
		Regexp: true,
		Expect: `parsekit\.TokenAPI\.Merge\(\): Merge\(\) called at ` +
			`/.*/tokenhandler_test\.go:\d+ on a non-forked TokenAPI`})
}