First working version of index query language (still in package sqe f…

…or now)

go.mod

@@ -73,6 +73,7 @@ require (
 require (
 	connectrpc.com/grpchealth v1.3.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
+	github.com/alecthomas/participle v0.7.1 // indirect
 	github.com/bobg/go-generics/v2 v2.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect

go.sum

@@ -115,6 +115,9 @@ github.com/abourget/llerrgroup v0.2.0/go.mod h1:QukSa1Sim/0R4aRlWdiBdAy+0i1PBfOd
 github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
 github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
 github.com/alecthomas/gometalinter v2.0.11+incompatible/go.mod h1:qfIpQGGz3d+NmgyPBqv+LSh50emm1pt72EtcX2vKYQk=
+github.com/alecthomas/participle v0.7.1 h1:2bN7reTw//5f0cugJcTOnY/NYZcWQOaajW+BwZB5xWs=
+github.com/alecthomas/participle v0.7.1/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=

pb/generate.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -u
-# Copyright 2019 dfuse Platform Inc.
+# Copyright 2024 StreamingFast Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

sqe/api.go

@@ -0,0 +1,54 @@
+package sqe
+
+import (
+	"context"
+	"fmt"
+)
+
+// FindAllFieldNames returns all used field names in the AST. There
+// is **NO** ordering on the elements, i.e. they might not come in the
+// same order specified in the AST.
+func ExtractAllKeys(expression Expression) (out []string) {
+	uniqueFieldNames := map[string]bool{}
+	onExpression := func(_ context.Context, expr Expression) error {
+		if v, ok := expr.(*KeyTerm); ok {
+			uniqueFieldNames[v.Value.Value] = true
+		}
+
+		return nil
+	}
+
+	visitor := NewDepthFirstVisitor(nil, onExpression)
+	expression.Visit(context.Background(), visitor)
+
+	i := 0
+	out = make([]string, len(uniqueFieldNames))
+	for fieldName := range uniqueFieldNames {
+		out[i] = fieldName
+		i++
+	}
+
+	return
+}
+
+func TransformExpression(expr Expression, transformer FieldTransformer) error {
+	if transformer == nil {
+		return nil
+	}
+
+	onExpression := func(_ context.Context, expr Expression) error {
+		v, ok := expr.(*KeyTerm)
+		if !ok {
+			return nil
+		}
+
+		if err := transformer.TransformStringLiteral("", v.Value); err != nil {
+			return fmt.Errorf("key %q transformation failed: %s", v.Value.Value, err)
+		}
+
+		return nil
+	}
+
+	visitor := NewDepthFirstVisitor(nil, onExpression)
+	return expr.Visit(context.Background(), visitor)
+}

sqe/api_test.go

@@ -0,0 +1,150 @@
+// Copyright 2024 StreamingFast Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqe
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// func TestExpressionToBleveQuery(t *testing.T) {
+// 	tests := []struct {
+// 		in          string
+// 		expectBleve string
+// 	}{
+// 		{
+// 			in:          "account:eoscanadacom",
+// 			expectBleve: `{"term":"eoscanadacom","field":"account"}`,
+// 		},
+// 		{
+// 			in:          "data.active:true",
+// 			expectBleve: `{"bool":true,"field":"data.active"}`,
+// 		},
+// 		{
+// 			in:          "data.active:false",
+// 			expectBleve: `{"bool":false,"field":"data.active"}`,
+// 		},
+// 		{
+// 			in:          `data.active:"true"`,
+// 			expectBleve: `{"term":"true","field":"data.active"}`,
+// 		},
+// 		{
+// 			in:          "receiver:eoscanadacom account:eoscanadacom",
+// 			expectBleve: `{"conjuncts":[{"term":"eoscanadacom","field":"receiver"},{"term":"eoscanadacom","field":"account"}]}`,
+// 		},
+// 		{
+// 			in:          "account:eoscanadacom receiver:eoscanadacom",
+// 			expectBleve: `{"conjuncts":[{"term":"eoscanadacom","field":"account"},{"term":"eoscanadacom","field":"receiver"}]}`,
+// 		},
+// 		{
+// 			in:          "receiver:eoscanadacom (action:transfer OR action:issue)",
+// 			expectBleve: `{"conjuncts":[{"term":"eoscanadacom","field":"receiver"},{"disjuncts":[{"term":"transfer","field":"action"},{"term":"issue","field":"action"}],"min":1}]}`,
+// 		},
+// 		{
+// 			in:          "receiver:eoscanadacom -(action:transfer OR action:issue)",
+// 			expectBleve: `{"conjuncts":[{"term":"eoscanadacom","field":"receiver"},{"must_not":{"disjuncts":[{"disjuncts":[{"term":"transfer","field":"action"},{"term":"issue","field":"action"}],"min":1}],"min":0}}]}`,
+// 		},
+// 		{
+// 			in:          "-receiver:eoscanadacom (action:transfer OR action:issue)",
+// 			expectBleve: `{"conjuncts":[{"must_not":{"disjuncts":[{"term":"eoscanadacom","field":"receiver"}],"min":0}},{"disjuncts":[{"term":"transfer","field":"action"},{"term":"issue","field":"action"}],"min":1}]}`,
+// 		},
+// 		{
+// 			in:          "-action:patate",
+// 			expectBleve: `{"must_not":{"disjuncts":[{"term":"patate","field":"action"}],"min":0}}`,
+// 		},
+// 		{
+// 			in: "receiver:eoscanadacom (action:transfer OR action:issue) account:eoscanadacom (data.from:eoscanadacom OR data.to:eoscanadacom)",
+// 			expectBleve: `{
+// 				"conjuncts": [
+// 				  { "term": "eoscanadacom", "field": "receiver" },
+// 				  { "disjuncts": [
+// 					  { "term": "transfer", "field": "action" },
+// 					  { "term": "issue", "field": "action" }
+// 					], "min": 1
+// 				  },
+// 				  { "term": "eoscanadacom", "field": "account" },
+// 				  { "disjuncts": [
+// 					  { "term": "eoscanadacom", "field": "data.from" },
+// 					  { "term": "eoscanadacom", "field": "data.to" }
+// 					], "min": 1
+// 				  }
+// 				]
+// 			  }`,
+// 		},
+// 	}
+
+// 	for idx, test := range tests {
+// 		t.Run(fmt.Sprintf("index %d", idx+1), func(t *testing.T) {
+// 			ast, err := Parse(context.Background(), test.in)
+// 			require.NoError(t, err)
+
+// 			res := ExpressionToBleve(ast)
+
+// 			cnt, err := json.Marshal(res)
+// 			require.NoError(t, err)
+// 			assert.JSONEq(t, test.expectBleve, string(cnt), "Failed on SQE %q, got %s", test.in, string(cnt))
+// 		})
+// 	}
+// }
+
+func TestExtractAllKeys(t *testing.T) {
+	tests := []struct {
+		in           string
+		expectedKeys []string
+	}{
+		{
+			"account",
+			[]string{"account"},
+		},
+		{
+			"data.active",
+			[]string{"data.active"},
+		},
+		{
+			"data.active",
+			[]string{"data.active"},
+		},
+		{
+			`"data.active"`,
+			[]string{"data.active"},
+		},
+		{
+			"receiver account",
+			[]string{"receiver", "account"},
+		},
+		{
+			"receiver (action || action)",
+			[]string{"receiver", "action"},
+		},
+		{
+			"receiver (action || action) account (data.from || data.to)",
+			[]string{"receiver", "action", "account", "data.from", "data.to"},
+		},
+	}
+
+	for idx, test := range tests {
+		t.Run(fmt.Sprintf("index %d", idx+1), func(t *testing.T) {
+			ast, err := Parse(context.Background(), test.in)
+			require.NoError(t, err)
+
+			actuals := ExtractAllKeys(ast)
+			assert.ElementsMatch(t, test.expectedKeys, actuals, "Mistmatch for SQE %q", test.in)
+		})
+	}
+}

sqe/errors.go

@@ -0,0 +1,39 @@
+package sqe
+
+import (
+	"fmt"
+
+	lex "github.com/alecthomas/participle/lexer"
+)
+
+type ParseError struct {
+	message  string
+	position lex.Position
+}
+
+func parserError(message string, position lex.Position) *ParseError {
+	return &ParseError{
+		message:  message,
+		position: position,
+	}
+}
+
+func rangeParserError(message string, start lex.Position, end lex.Position) *ParseError {
+	return &ParseError{
+		message: message,
+		position: lex.Position{
+			Filename: start.Filename,
+			Offset:   start.Offset,
+			Line:     start.Line,
+			Column:   end.Column,
+		},
+	}
+}
+
+func (e *ParseError) Error() string {
+	if e.position.Line <= 1 {
+		return fmt.Sprintf("%s at column %d", e.message, e.position.Offset)
+	}
+
+	return fmt.Sprintf("%s at line %d column %d", e.message, e.position.Line, e.position.Column)
+}

sqe/init_test.go

@@ -0,0 +1,79 @@
+package sqe
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+)
+
+func expressionToString(expression Expression) string {
+	builder := &strings.Builder{}
+	visitor := &TestVisitor{
+		writer: builder,
+	}
+
+	expression.Visit(context.Background(), visitor)
+
+	return builder.String()
+}
+
+type TestVisitor struct {
+	writer io.Writer
+}
+
+func (v *TestVisitor) Visit_And(ctx context.Context, e *AndExpression) error {
+	return v.visit_binary(ctx, "<", "&&", ">", e.Children)
+}
+
+func (v *TestVisitor) Visit_Or(ctx context.Context, e *OrExpression) error {
+	return v.visit_binary(ctx, "[", "||", "]", e.Children)
+}
+
+func (v *TestVisitor) visit_binary(ctx context.Context, opStart, op, opEnd string, children []Expression) error {
+	v.print(opStart)
+
+	for i, child := range children {
+		if i != 0 {
+			v.print(" %s ", op)
+		}
+
+		child.Visit(ctx, v)
+	}
+	v.print(opEnd)
+
+	return nil
+}
+
+func (v *TestVisitor) Visit_Parenthesis(ctx context.Context, e *ParenthesisExpression) error {
+	v.print("(")
+	e.Child.Visit(ctx, v)
+	v.print(")")
+
+	return nil
+}
+
+func (v *TestVisitor) Visit_Not(ctx context.Context, e *NotExpression) error {
+	v.print("!")
+	e.Child.Visit(ctx, v)
+
+	return nil
+}
+
+func (v *TestVisitor) Visit_KeyTerm(ctx context.Context, e *KeyTerm) error {
+	v.printStringLiteral(e.Value)
+	return nil
+}
+
+func (v *TestVisitor) printStringLiteral(literal *StringLiteral) error {
+	if literal.QuotingChar != "" {
+		return v.print("%s%s%s", literal.QuotingChar, literal.Value, literal.QuotingChar)
+	}
+
+	return v.print(literal.Value)
+}
+
+func (v *TestVisitor) print(message string, args ...interface{}) error {
+	fmt.Fprintf(v.writer, message, args...)
+	return nil
+}