Skip to content

Commit

Permalink
✨ Streamline AWS discovery filters (#4776)
Browse files Browse the repository at this point in the history 
* Refactored AWS discovery filters and made regions global instead of EC2 specific.

Signed-off-by: Vasil Sirakov <[email protected]>

* Rename type.

Signed-off-by: Vasil Sirakov <[email protected]>

* Use more specific filter type in function.

Signed-off-by: Vasil Sirakov <[email protected]>

* Fix unit tests.

Signed-off-by: Vasil Sirakov <[email protected]>

* Remove debug logs.

Signed-off-by: Vasil Sirakov <[email protected]>

* Use instance-ids instead of iid for EC2 filter keys.

Signed-off-by: Vasil Sirakov <[email protected]>

---------

Signed-off-by: Vasil Sirakov <[email protected]>
  • Loading branch information
@VasilSirakov
VasilSirakov authored Oct 24, 2024
1 parent 1fb2db8 commit ae2cae0
Show file tree
Hide file tree
Showing 9 changed files with 137 additions and 291 deletions.
87 changes: 48 additions & 39 deletions providers/aws/connection/connection.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package connection
import (
"context"
"errors"
"slices"
"strconv"
"strings"

Expand Down Expand Up @@ -37,33 +38,43 @@ type AwsConnection struct {
PlatformOverride string
connectionOptions map[string]string
Filters DiscoveryFilters
RegionLimits []string
scope string
}

type DiscoveryFilters struct {
Ec2DiscoveryFilters Ec2DiscoveryFilters
EcrDiscoveryFilters EcrDiscoveryFilters
EcsDiscoveryFilters EcsDiscoveryFilters
GeneralDiscoveryFilters GeneralResourceDiscoveryFilters
Ec2DiscoveryFilters Ec2DiscoveryFilters
EcrDiscoveryFilters EcrDiscoveryFilters
EcsDiscoveryFilters EcsDiscoveryFilters
DiscoveryFilters GeneralDiscoveryFilters
}

// ensure all underlying reference types aren't `nil`
func EmptyDiscoveryFilters() DiscoveryFilters {
return DiscoveryFilters{
DiscoveryFilters: GeneralDiscoveryFilters{Regions: []string{}, ExcludeRegions: []string{}},
Ec2DiscoveryFilters: Ec2DiscoveryFilters{InstanceIds: []string{}, ExcludeInstanceIds: []string{}, Tags: map[string]string{}, ExcludeTags: map[string]string{}},
EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}, ExcludeTags: []string{}},
EcsDiscoveryFilters: EcsDiscoveryFilters{},
}
}

type GeneralResourceDiscoveryFilters struct {
Tags map[string]string
Regions []string
type GeneralDiscoveryFilters struct {
Regions []string
ExcludeRegions []string
}

type Ec2DiscoveryFilters struct {
Regions []string
Tags map[string]string
InstanceIds []string
ExcludeRegions []string
ExcludeTags map[string]string
ExcludeInstanceIds []string
Tags map[string]string
ExcludeTags map[string]string
}

type EcrDiscoveryFilters struct {
Tags []string
Tags []string
ExcludeTags []string
}

type EcsDiscoveryFilters struct {
OnlyRunningContainers bool
DiscoverImages bool
Expand All @@ -82,6 +93,7 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
// check flags for connection options
c := &AwsConnection{
awsConfigOptions: []func(*config.LoadOptions) error{},
Filters: EmptyDiscoveryFilters(),
}
opts := parseFlagsForConnectionOptions(asset.Options, conf.Options, conf.GetCredentials())
for _, opt := range opts {
Expand Down Expand Up @@ -125,38 +137,30 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
c.connectionOptions = asset.Options
if conf.Discover != nil {
c.Filters = parseOptsToFilters(conf.Discover.Filter)
c.RegionLimits = c.Filters.GeneralDiscoveryFilters.Regions
}
return c, nil
}

func parseOptsToFilters(opts map[string]string) DiscoveryFilters {
d := DiscoveryFilters{
Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}},
EcsDiscoveryFilters: EcsDiscoveryFilters{},
EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}},
GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}},
}
d := EmptyDiscoveryFilters()
for k, v := range opts {
switch {
case strings.HasPrefix(k, "ec2:tag:"):
d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v
case strings.HasPrefix(k, "exclude:ec2:tag:"):
d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "exclude:ec2:tag:")] = v
case k == "ec2:regions":
d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, strings.Split(v, ",")...)
case k == "exclude:ec2:regions":
d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, strings.Split(v, ",")...)
case k == "all:regions", k == "regions":
d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, strings.Split(v, ",")...)
case k == "regions":
d.DiscoveryFilters.Regions = append(d.DiscoveryFilters.Regions, strings.Split(v, ",")...)
case k == "exclude:regions":
d.DiscoveryFilters.ExcludeRegions = append(d.DiscoveryFilters.ExcludeRegions, strings.Split(v, ",")...)
case k == "ec2:instance-ids":
d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, strings.Split(v, ",")...)
case k == "exclude:ec2:instance-ids":
case k == "ec2:exclude:instance-ids":
d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, strings.Split(v, ",")...)
case strings.HasPrefix(k, "all:tag:"):
d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v
case strings.HasPrefix(k, "ec2:tag:"):
d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v
case strings.HasPrefix(k, "ec2:exclude:tag:"):
d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "ec2:exclude:tag:")] = v
case k == "ecr:tags":
d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, strings.Split(v, ",")...)
case k == "ecr:exclude:tags":
d.EcrDiscoveryFilters.ExcludeTags = append(d.EcrDiscoveryFilters.ExcludeTags, strings.Split(v, ",")...)
case k == "ecs:only-running-containers":
parsed, err := strconv.ParseBool(v)
if err == nil {
Expand Down Expand Up @@ -360,15 +364,17 @@ func (h *AwsConnection) Regions() ([]string, error) {
log.Debug().Msg("use regions from cache")
return c.Data.([]string), nil
}
log.Debug().Msg("no region cache found. fetching regions")

if len(h.RegionLimits) > 0 {
log.Debug().Interface("regions", h.RegionLimits).Msg("using region limits")
// include filters have precedense over exclude filters. in any normal situation they should be mutually exclusive.
regionLimits := h.Filters.DiscoveryFilters.Regions
if len(regionLimits) > 0 {
log.Debug().Interface("regions", regionLimits).Msg("using region limits")
// cache the regions as part of the provider instance
h.clientcache.Store("_regions", &CacheEntry{Data: h.RegionLimits})
return h.RegionLimits, nil
h.clientcache.Store("_regions", &CacheEntry{Data: regionLimits})
return regionLimits, nil
}
// if no cache, get regions using ec2 client (using the ssm list global regions does not give the same list)
log.Debug().Msg("no region cache or region limits found. fetching regions")
regions := []string{}
svc := h.Ec2("us-east-1")
ctx := context.Background()
Expand All @@ -383,7 +389,10 @@ func (h *AwsConnection) Regions() ([]string, error) {
}
}
for _, region := range res.Regions {
regions = append(regions, *region.RegionName)
// ensure excluded regions are discarded
if !slices.Contains(h.Filters.DiscoveryFilters.ExcludeRegions, *region.RegionName) {
regions = append(regions, *region.RegionName)
}
}
// cache the regions as part of the provider instance
h.clientcache.Store("_regions", &CacheEntry{Data: regions})
Expand Down
95 changes: 26 additions & 69 deletions providers/aws/connection/connection_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,110 +9,67 @@ import (
"github.com/stretchr/testify/require"
)

// testParseOptsToFilters accepts a map which doesn't guarantee a deterministic iteration order. this means that slices
// in the parsed filters need to be compared individually ensuring their elements match regardless of their order.
func compareFilters(t *testing.T, expected, actual DiscoveryFilters) {
require.ElementsMatch(t, expected.Ec2DiscoveryFilters.Regions, actual.Ec2DiscoveryFilters.Regions)
require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeRegions, actual.Ec2DiscoveryFilters.ExcludeRegions)

require.ElementsMatch(t, expected.Ec2DiscoveryFilters.InstanceIds, actual.Ec2DiscoveryFilters.InstanceIds)
require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeInstanceIds, actual.Ec2DiscoveryFilters.ExcludeInstanceIds)

require.Equal(t, expected.Ec2DiscoveryFilters.Tags, actual.Ec2DiscoveryFilters.Tags)
require.Equal(t, expected.Ec2DiscoveryFilters.ExcludeTags, actual.Ec2DiscoveryFilters.ExcludeTags)

require.Equal(t, expected.EcsDiscoveryFilters, actual.EcsDiscoveryFilters)

require.ElementsMatch(t, expected.EcrDiscoveryFilters.Tags, actual.EcrDiscoveryFilters.Tags)

require.ElementsMatch(t, expected.GeneralDiscoveryFilters.Regions, actual.GeneralDiscoveryFilters.Regions)
require.Equal(t, expected.GeneralDiscoveryFilters.Tags, actual.GeneralDiscoveryFilters.Tags)
}

func TestParseOptsToFilters(t *testing.T) {
t.Run("all opts are mapped to discovery filters correctly", func(t *testing.T) {
opts := map[string]string{
// DiscoveryFilters.Regions
"regions": "us-east-1,us-west-1,eu-west-1",
// DiscoveryFilters.ExcludeRegions
"exclude:regions": "us-east-2,us-west-2,eu-west-2",
// Ec2DiscoveryFilters.InstanceIds
"ec2:instance-ids": "iid-1,iid-2",
// Ec2DiscoveryFilters.ExcludeInstanceIds
"ec2:exclude:instance-ids": "iid-1,iid-2",
// Ec2DiscoveryFilters.Tags
"ec2:tag:key1": "val1",
"ec2:tag:key2": "val2",
// Ec2DiscoveryFilters.ExcludeTags
"exclude:ec2:tag:key1": "val1",
"exclude:ec2:tag:key2": "val2",
// Ec2DiscoveryFilters.Regions
"ec2:regions": "us-east-1,us-west-1",
// Ec2DiscoveryFilters.ExcludeRegions
"exclude:ec2:regions": "us-east-1,us-west-1",
// Ec2DiscoveryFilters.InstanceIds
"ec2:instance-ids": "iid-1,iid-2",
// Ec2DiscoveryFilters.ExcludeInstanceIds
"exclude:ec2:instance-ids": "iid-1,iid-2",
// GeneralDiscoveryFilters.Regions
"all:regions": "us-east-1,us-west-1,eu-west-1",
// GeneralDiscoveryFilters.Tags
"all:tag:key1": "val1",
"all:tag:key2": "val2",
"ec2:exclude:tag:key1": "val1,val2",
"ec2:exclude:tag:key2": "val3",
// EcrDiscoveryFilters.Tags
"ecr:tags": "tag1,tag2",
// EcrDiscoveryFilters.ExcludeTags
"ecr:exclude:tags": "tag1,tag2",
// EcsDiscoveryFilters
"ecs:only-running-containers": "true",
"ecs:discover-images": "T",
"ecs:discover-instances": "false",
}
expected := DiscoveryFilters{
DiscoveryFilters: GeneralDiscoveryFilters{
Regions: []string{"us-east-1", "us-west-1", "eu-west-1"},
ExcludeRegions: []string{"us-east-2", "us-west-2", "eu-west-2"},
},
Ec2DiscoveryFilters: Ec2DiscoveryFilters{
Regions: []string{
"us-east-1", "us-west-1",
},
ExcludeRegions: []string{
"us-east-1", "us-west-1",
},
InstanceIds: []string{
"iid-1", "iid-2",
},
ExcludeInstanceIds: []string{
"iid-1", "iid-2",
},
InstanceIds: []string{"iid-1", "iid-2"},
ExcludeInstanceIds: []string{"iid-1", "iid-2"},
Tags: map[string]string{
"key1": "val1",
"key2": "val2",
},
ExcludeTags: map[string]string{
"key1": "val1",
"key2": "val2",
"key1": "val1,val2",
"key2": "val3",
},
},
EcsDiscoveryFilters: EcsDiscoveryFilters{
OnlyRunningContainers: true,
DiscoverImages: true,
DiscoverInstances: false,
},
EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{
"tag1", "tag2",
}},
GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{
Regions: []string{
"us-east-1", "us-west-1", "eu-west-1",
},
Tags: map[string]string{
"key1": "val1",
"key2": "val2",
},
EcrDiscoveryFilters: EcrDiscoveryFilters{
Tags: []string{"tag1", "tag2"},
ExcludeTags: []string{"tag1", "tag2"},
},
}

actual := parseOptsToFilters(opts)
compareFilters(t, expected, actual)
require.Equal(t, expected, actual)
})

t.Run("empty opts are mapped to discovery filters correctly", func(t *testing.T) {
expected := DiscoveryFilters{
Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}},
EcsDiscoveryFilters: EcsDiscoveryFilters{},
EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}},
GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}},
}

expected := EmptyDiscoveryFilters()
actual := parseOptsToFilters(map[string]string{})
compareFilters(t, expected, actual)
require.Equal(t, expected, actual)
})
}
16 changes: 9 additions & 7 deletions providers/aws/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,18 @@ func parseFlagsToFiltersOpts(m map[string]*llx.Primitive) map[string]string {

if x, ok := m["filters"]; ok && len(x.Map) != 0 {
knownTagPrefixes := []string{
"ec2:tag:",
"exclude:ec2:tag:",
"ec2:regions",
"exclude:ec2:regions",
"all:regions",
// general filters
"regions",
"exclude:regions",
// ec2 filters
"ec2:instance-ids",
"exclude:ec2:instance-ids",
"all:tag:",
"ec2:exclude:instance-ids",
"ec2:tag:",
"ec2:exclude:tag:",
// ecr filters
"ecr:tags",
"ecr:exclude:tags",
// ecs filters
"ecs:only-running-containers",
"ecs:discover-instances",
"ecs:discover-images",
Expand Down
Loading

0 comments on commit ae2cae0

Please sign in to comment.