Issue with a nested structure

[gwd]

First, thank you so much for this library -- made my first grammar on Friday and I think the setup really helped make things straightforward.

I'm using participle v2.10.0, and trying to write a parser for strings like the following:

	tests := []string{
		"Xen 4.18.x",
		"Linux 4.7.x",
		"QEMU 4.7-RC series",
		"Linux 4.7.x - Linux 4.9.x",
		"xen-unstable",
		"Xen 4.18.x - xen-unstable",
		"QEMU 4.7-RC series - QEMU 4.10.x",
	}

i.e., a version that may be a single version, or a range (separated by -); that may be a single string, or <project> <version number>.

I wrote the following participle structures:

type ProjectVersion struct {
	Project string `parser:"@Project"`
	Version string `parser:"Space @(VersionString | RCVersionString)"`
}

type VersionSingle struct {
	TipVersion     *string         `parser:"@TipVersion"`
	ProjectVersion *ProjectVersion `parser:"| @@"`
}

type VersionRange struct {
	From VersionSingle `parser:"@@"`
	To   VersionSingle `parser:"VersionDash @@"`
}

type Version struct {
	Range  *VersionRange  `parser:"@@"`
	Single *VersionSingle `parser:"| @@"`
}

var rulesCommon []lexer.SimpleRule{	
	{"Project", `Xen|Linux|QEMU|xapi`},
	{"TipVersion", `xen-unstable`},
	{"RCVersionString", `\d+\.\d+-RC series`},
	{"VersionString", `\d+\.\d+\.x`},
	{"VersionDash", " - "},
	{"Space", ` `},
}

Unfortunately, I get errors like the following:

Parsing Xen 4.18.x: 1:11: unexpected token "<EOF>" (expected <versiondash> VersionSingle)
Parsing Linux 4.7.x: 1:12: unexpected token "<EOF>" (expected <versiondash> VersionSingle)
Parsing QEMU 4.7-RC series: 1:19: unexpected token "<EOF>" (expected <versiondash> VersionSingle)

In other words, it's somehow getting stuck on parsing something as a VersionRange, and not backing out and parsing it simply as a VersionSingle.

But this only happens if both Version and VersionSingle have at least two ways to be interpreted. If I replace the ProjectVersion with a single regexp that matches the same string, it works (here replacing the Version lexer token with a ProjectVersion token with the appropriate regexp).

// With the range, without the "project version"
type VersionSingleNoProjectVersion struct {
	TipVersion     *string `parser:"@TipVersion"`
	ProjectVersion *string `parser:"| @ProjectVersion"`
}

type VersionRangeNoProjectVersion struct {
	From VersionSingleNoProjectVersion `parser:"@@"`
	To   VersionSingleNoProjectVersion `parser:"VersionDash @@"`
}

type VersionNoProjectVersion struct {
	Range  *VersionRangeNoProjectVersion  `parser:"@@"`
	Single *VersionSingleNoProjectVersion `parser:"| @@"`
}

The problem for single items goes away if I get rid of the RangeVersion; but then of course you can't parse ranges:

// With "project version", without range
type VersionNoRange struct {
	Single *VersionSingle `parser:"@@"`
}

Any idea what's going on?

For completeness, here's a complete testing function you can use to trigger the issue:

package participletest_test

import (
	"testing"

	"github.com/alecthomas/participle/v2"
	"github.com/alecthomas/participle/v2/lexer"
)

// What I'd like:
type ProjectVersion struct {
	Project string `parser:"@Project"`
	Version string `parser:"Space @(VersionString | RCVersionString)"`
}

type VersionSingle struct {
	TipVersion     *string         `parser:"@TipVersion"`
	ProjectVersion *ProjectVersion `parser:"| @@"`
}

type VersionRange struct {
	From VersionSingle `parser:"@@"`
	To   VersionSingle `parser:"VersionDash @@"`
}

type Version struct {
	Range  *VersionRange  `parser:"@@"`
	Single *VersionSingle `parser:"| @@"`
}

// With "project version", without range
type VersionNoRange struct {
	Single *VersionSingle `parser:"@@"`
}

// With the range, without the "project version"
type VersionSingleNoProjectVersion struct {
	TipVersion     *string `parser:"@TipVersion"`
	ProjectVersion *string `parser:"| @ProjectVersion"`
}

type VersionRangeNoProjectVersion struct {
	From VersionSingleNoProjectVersion `parser:"@@"`
	To   VersionSingleNoProjectVersion `parser:"VersionDash @@"`
}

type VersionNoProjectVersion struct {
	Range  *VersionRangeNoProjectVersion  `parser:"@@"`
	Single *VersionSingleNoProjectVersion `parser:"| @@"`
}

var rulesCommon = []lexer.SimpleRule{
	{"TipVersion", `xen-unstable`},
	{"RCVersionString", `\d+\.\d+-RC series`},
	{"VersionString", `\d+\.\d+\.x`},
	{"VersionDash", " - "},
	{"Space", ` `},
}

func TestVersion(t *testing.T) {
	simpletests := []string{
		"Xen 4.18.x",
		"Linux 4.7.x",
		"QEMU 4.7-RC series",
		"xen-unstable",
	}

	rangetests := []string{
		"Linux 4.7.x - Linux 4.9.x",
		"Xen 4.18.x - xen-unstable",
		"QEMU 4.7-RC series - QEMU 4.10.x",
	}

	lexProject := lexer.MustSimple(append(rulesCommon, lexer.SimpleRule{"Project", `Xen|Linux|QEMU|xapi`}))

	lexProjectVersion := lexer.MustSimple(append(rulesCommon,
		lexer.SimpleRule{"ProjectVersion", `(Xen|Linux|QEMU|xapi) (\d+\.\d+\.x|\d+\.\d+-RC series)`}))

	pVersion := participle.MustBuild[Version](participle.Lexer(lexProject))

	t.Log("Testing pVersion with simple and range")
	for _, in := range append(simpletests, rangetests...) {
		out, err := pVersion.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}

	pVersionNoRange := participle.MustBuild[VersionNoRange](participle.Lexer(lexProject))

	t.Log("Testing pVersionNoRange with simple only")
	for _, in := range simpletests {
		out, err := pVersionNoRange.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}

	pVersionNoProjcetVersion := participle.MustBuild[VersionNoProjectVersion](participle.Lexer(lexProjectVersion))

	t.Log("Testing pVersionNoProjectVersion with simple and range")
	for _, in := range append(simpletests, rangetests...) {
		out, err := pVersionNoProjcetVersion.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}
}

[gwd]

BTW, the following works for all tests:

type VersionSlice struct {
	Range []VersionSingle `parser:"@@ ( VersionDash @@ )*"`
}

...
	pVersionSlice := participle.MustBuild[VersionSlice](participle.Lexer(lexProject))

	t.Log("Testing pVersionSlice with simple and range")
	for _, in := range append(simpletests, rangetests...) {
		out, err := pVersionSlice.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}
...

So I think I have a work-around for now; but if the first version is supposed to work, it would be good to track down what's going on.

[alecthomas]

I'm not sure if this is still an issue, or what the exact problem is, but one thing that stands out to me is that your lexer is too complex. It's best to keep your lexer very straightforward, and only match very simple tokens. For example I would expect something more like this:

var rulesCommon = []lexer.SimpleRule{
	{"Ident", `\w+`},
	{"Number", `\d+`},
	{"Punct", "[-.]"}, // Maybe more?
	{"Space", ` `},
}

Then move the more complex matching into the parser:

type ProjectVersion struct {
	Project string `parser:"@('Xen'|'Linux'|'QEMU'|'xapi')'"`
	Version string `parser:"Space @(....)"`
}

[gwd]

Thanks for the response. Wouldn't your suggestion, however, mean that the parser would accept non-standard version strings (e.g., "Xen 4" instead of "Xen 4.3.x"? Part of the goal of this was to enforce regularity so that the results could be machine-parseable.

[alecthomas]

No, you'd just match the specific version structure you want in the parser rather than the lexer, eg.@(Number '.' Number '.' Number)