From 309c066603023d7f6c1ccd35528f5dcaf98538f2 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Fri, 5 Jan 2024 22:20:28 -0800 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=20map.all/none/one/contains=20(#2954)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similar to arrays, we now support this vital set of operations for maps. This is particularly useful for tags. Note: dicts are not yet supported and will be added next. A few examples: ```coffee {'a': 1, 'b': 2}.contains( key == 'b' ) {'a': 1, 'b': 2}.all( value > 0 ) {'a': 1, 'b': 2}.one( value != 1 ) {'a': 1, 'b': 2}.none( key == /d-f/ ) ``` Signed-off-by: Dominik Richter --- llx/builtin.go | 4 + llx/builtin_map.go | 52 ++++++++++++ mqlc/builtin.go | 16 ++-- mqlc/builtin_map.go | 116 +++++++++++++++++++++++++-- providers/core/resources/mql_test.go | 75 +++++++++++++++++ 5 files changed, 251 insertions(+), 12 deletions(-) diff --git a/llx/builtin.go b/llx/builtin.go index ed033078ae..af97c461fc 100644 --- a/llx/builtin.go +++ b/llx/builtin.go @@ -647,6 +647,10 @@ func init() { "length": {f: mapLengthV2}, "where": {f: mapWhereV2}, "$whereNot": {f: mapWhereNotV2}, + "$any": {f: mapAny}, + "$one": {f: mapOne}, + "$none": {f: mapNone}, + "$all": {f: mapAll}, "{}": {f: mapBlockCallV2}, "keys": {f: mapKeysV2, Label: "keys"}, "values": {f: mapValuesV2, Label: "values"}, diff --git a/llx/builtin_map.go b/llx/builtin_map.go index 5929c66524..c4e5f4c06f 100644 --- a/llx/builtin_map.go +++ b/llx/builtin_map.go @@ -196,6 +196,58 @@ func mapWhereNotV2(e *blockExecutor, bind *RawData, chunk *Chunk, ref uint64) (* return _mapWhereV2(e, bind, chunk, ref, true) } +func mapAll(e *blockExecutor, bind *RawData, chunk *Chunk, ref uint64) (*RawData, uint64, error) { + if bind.Value == nil { + return &RawData{Type: types.Bool, Error: errors.New("failed to validate all entries (map is null)")}, 0, nil + } + + filteredList := bind.Value.(map[string]interface{}) + + if len(filteredList) != 0 { + return BoolFalse, 0, nil + } + return BoolTrue, 0, nil +} + +func mapNone(e *blockExecutor, bind *RawData, chunk *Chunk, ref uint64) (*RawData, uint64, error) { + if bind.Value == nil { + return &RawData{Type: types.Bool, Error: errors.New("failed to validate all entries (map is null)")}, 0, nil + } + + filteredList := bind.Value.(map[string]interface{}) + + if len(filteredList) != 0 { + return BoolFalse, 0, nil + } + return BoolTrue, 0, nil +} + +func mapAny(e *blockExecutor, bind *RawData, chunk *Chunk, ref uint64) (*RawData, uint64, error) { + if bind.Value == nil { + return &RawData{Type: types.Bool, Error: errors.New("failed to validate all entries (map is null)")}, 0, nil + } + + filteredList := bind.Value.(map[string]interface{}) + + if len(filteredList) == 0 { + return BoolFalse, 0, nil + } + return BoolTrue, 0, nil +} + +func mapOne(e *blockExecutor, bind *RawData, chunk *Chunk, ref uint64) (*RawData, uint64, error) { + if bind.Value == nil { + return &RawData{Type: types.Bool, Error: errors.New("failed to validate all entries (map is null)")}, 0, nil + } + + filteredList := bind.Value.(map[string]interface{}) + + if len(filteredList) != 1 { + return BoolFalse, 0, nil + } + return BoolTrue, 0, nil +} + func mapBlockCallV2(e *blockExecutor, bind *RawData, chunk *Chunk, ref uint64) (*RawData, uint64, error) { return e.runBlock(bind, chunk.Function.Args[0], nil, ref) } diff --git a/mqlc/builtin.go b/mqlc/builtin.go index fceadfb9fa..51f9df6799 100644 --- a/mqlc/builtin.go +++ b/mqlc/builtin.go @@ -111,12 +111,16 @@ func init() { "flat": {compile: compileArrayFlat, signature: FunctionSignature{}}, }, types.MapLike: { - "[]": {typ: childType, signature: FunctionSignature{Required: 1, Args: []types.Type{types.String}}}, - "{}": {typ: blockType, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, - "length": {typ: intType, signature: FunctionSignature{}}, - "where": {compile: compileMapWhere, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, - "keys": {typ: stringArrayType, signature: FunctionSignature{}}, - "values": {compile: compileMapValues, signature: FunctionSignature{}}, + "[]": {typ: childType, signature: FunctionSignature{Required: 1, Args: []types.Type{types.String}}}, + "{}": {typ: blockType, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, + "length": {typ: intType, signature: FunctionSignature{}}, + "keys": {typ: stringArrayType, signature: FunctionSignature{}}, + "values": {compile: compileMapValues, signature: FunctionSignature{}}, + "where": {compile: compileMapWhere, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, + "contains": {compile: compileMapContains, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, + "all": {compile: compileMapAll, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, + "one": {compile: compileMapOne, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, + "none": {compile: compileMapNone, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}}, }, types.ResourceLike: { // "": compileHandler{compile: compileResourceDefault}, diff --git a/mqlc/builtin_map.go b/mqlc/builtin_map.go index e01c1cc1d8..1ac2d7283e 100644 --- a/mqlc/builtin_map.go +++ b/mqlc/builtin_map.go @@ -439,6 +439,19 @@ func compileDictFlat(c *compiler, typ types.Type, ref uint64, id string, call *p return typ, nil } +func compileMapValues(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) { + typ = types.Array(typ.Child()) + c.addChunk(&llx.Chunk{ + Call: llx.Chunk_FUNCTION, + Id: id, + Function: &llx.Function{ + Type: string(typ), + Binding: ref, + }, + }) + return typ, nil +} + func compileMapWhere(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) { if call == nil { return types.Nil, errors.New("missing filter argument for calling '" + id + "'") @@ -520,15 +533,106 @@ func compileMapWhere(c *compiler, typ types.Type, ref uint64, id string, call *p return typ, nil } -func compileMapValues(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) { - typ = types.Array(typ.Child()) +func compileMapContains(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) { + _, err := compileMapWhere(c, typ, ref, "where", call) + if err != nil { + return types.Nil, err + } + listRef := c.tailRef() + + if err := compileListAssertionMsg(c, typ, ref, listRef, listRef); err != nil { + return types.Nil, err + } + c.addChunk(&llx.Chunk{ Call: llx.Chunk_FUNCTION, - Id: id, + Id: "$any", Function: &llx.Function{ - Type: string(typ), - Binding: ref, + Type: string(types.Bool), + Binding: listRef, }, }) - return typ, nil + + checksum := c.Result.CodeV2.Checksums[c.tailRef()] + c.Result.Labels.Labels[checksum] = "[].contains()" + + return types.Bool, nil +} + +func compileMapAll(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) { + _, err := compileMapWhere(c, typ, ref, "$whereNot", call) + if err != nil { + return types.Nil, err + } + listRef := c.tailRef() + + if err := compileListAssertionMsg(c, typ, ref, listRef, listRef); err != nil { + return types.Nil, err + } + + c.addChunk(&llx.Chunk{ + Call: llx.Chunk_FUNCTION, + Id: "$all", + Function: &llx.Function{ + Type: string(types.Bool), + Binding: listRef, + }, + }) + + checksum := c.Result.CodeV2.Checksums[c.tailRef()] + c.Result.Labels.Labels[checksum] = "[].all()" + + return types.Bool, nil +} + +func compileMapOne(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) { + _, err := compileMapWhere(c, typ, ref, "where", call) + if err != nil { + return types.Nil, err + } + listRef := c.tailRef() + + if err := compileListAssertionMsg(c, typ, ref, listRef, listRef); err != nil { + return types.Nil, err + } + + c.addChunk(&llx.Chunk{ + Call: llx.Chunk_FUNCTION, + Id: "$one", + Function: &llx.Function{ + Type: string(types.Bool), + Binding: listRef, + }, + }) + + checksum := c.Result.CodeV2.Checksums[c.tailRef()] + c.Result.Labels.Labels[checksum] = "[].one()" + + return types.Bool, nil +} + +func compileMapNone(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) { + _, err := compileMapWhere(c, typ, ref, "where", call) + if err != nil { + return types.Nil, err + } + listRef := c.tailRef() + + if err := compileListAssertionMsg(c, typ, ref, listRef, listRef); err != nil { + return types.Nil, err + } + + c.addChunk(&llx.Chunk{ + Call: llx.Chunk_FUNCTION, + Id: "$none", + Function: &llx.Function{ + Type: string(types.Bool), + Binding: listRef, + }, + }) + + checksum := c.Result.CodeV2.Checksums[c.tailRef()] + c.Result.Labels.Labels[checksum] = "[].none()" + + return types.Bool, nil } diff --git a/providers/core/resources/mql_test.go b/providers/core/resources/mql_test.go index bc0331a188..70f6a26032 100644 --- a/providers/core/resources/mql_test.go +++ b/providers/core/resources/mql_test.go @@ -688,6 +688,81 @@ func TestArray(t *testing.T) { }) } +func TestMap(t *testing.T) { + m := "{'a': 1, 'b': 1, 'c': 2}" + x := testutils.InitTester(testutils.LinuxMock()) + x.TestSimple(t, []testutils.SimpleTest{ + // contains + { + Code: m + ".contains(key == 'a')", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".contains(key == 'z')", + ResultIndex: 1, Expectation: false, + }, + { + Code: m + ".contains(value == 1)", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".contains(value == 0)", + ResultIndex: 1, Expectation: false, + }, + // all + { + Code: m + ".all(key == /[abc]/)", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".all(key == 'a')", + ResultIndex: 1, Expectation: false, + }, + { + Code: m + ".all(value > 0)", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".all(value == 0)", + ResultIndex: 1, Expectation: false, + }, + // none + { + Code: m + ".none(key == /[m-z]/)", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".none(key == /[b-z]/)", + ResultIndex: 1, Expectation: false, + }, + { + Code: m + ".none(value < 1)", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".none(value <= 2)", + ResultIndex: 1, Expectation: false, + }, + // one + { + Code: m + ".one(key == 'a')", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".one(key == /[a-b]/)", + ResultIndex: 1, Expectation: false, + }, + { + Code: m + ".one(value == 2)", + ResultIndex: 1, Expectation: true, + }, + { + Code: m + ".one(value == 1)", + ResultIndex: 1, Expectation: false, + }, + }) +} + func TestResource_Default(t *testing.T) { x := testutils.InitTester(testutils.LinuxMock()) res := x.TestQuery(t, "mondoo")