Skip to content

Commit

Permalink
🐛 fix unset fromPort/toPort and port on aws resources (#3594)
Browse files Browse the repository at this point in the history 
When the fromPort and toPort on security groups are not set, they are not internally treated as 0, but rather -1 (which is similar to unset). Technically we could even expose the unset state, but we should reserve that for the future.

I also updated a few other occurances of port to make it show up as -1. The code for number handling in AWS responses has been further simplified.

Signed-off-by: Dominik Richter <[email protected]>
  • Loading branch information
@arlimus
arlimus authored Mar 19, 2024
1 parent 1d2a7a3 commit ae88e62
Show file tree
Hide file tree
Showing 23 changed files with 83 additions and 98 deletions.
14 changes: 11 additions & 3 deletions llx/rawdata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,21 +438,29 @@ var BoolFalse = BoolData(false)
var BoolTrue = BoolData(true)

// IntData creates a rawdata struct from a go int
func IntData(v int64) *RawData {
func IntData[T int64 | int32 | int](v T) *RawData {
return &RawData{
Type: types.Int,
Value: v,
Value: int64(v),
}
}

// IntDataPtr creates a rawdata struct from a go int pointer
func IntDataPtr(v *int64) *RawData {
func IntDataPtr[T int64 | int32 | int](v *T) *RawData {
if v == nil {
return NilData
}
return IntData(*v)
}

// IntDataDefault creates a rawdata struct from a go int pointer with a fallback default value
func IntDataDefault[T int64 | int32 | int](v *T, dflt int64) *RawData {
if v == nil {
return IntData(dflt)
}
return IntData(*v)
}

// FloatData creates a rawdata struct from a go float
func FloatData(v float64) *RawData {
return &RawData{
Expand Down
2 changes: 1 addition & 1 deletion llx/rawdata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestRawData_String(t *testing.T) {
{BoolDataPtr(nil), "<null>"},
{IntData(0), "0"},
{IntDataPtr(&intVal), "1"},
{IntDataPtr(nil), "<null>"},
{IntDataPtr[int](nil), "<null>"},
{FloatData(123), "123"},
{StringData("yo"), "\"yo\""},
{StringDataPtr(nil), "<null>"},
Expand Down
21 changes: 0 additions & 21 deletions providers-sdk/v1/util/convert/pointers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,27 +24,6 @@ func ToString(ptr *string) string {
return *ptr
}

func ToInt64(ptr *int64) int64 {
if ptr == nil {
return 0
}
return *ptr
}

func ToInt64From32(ptr *int32) int64 {
if ptr == nil {
return 0
}
return int64(*ptr)
}

func ToInt64FromInt(ptr *int) int64 {
if ptr == nil {
return 0
}
return int64(*ptr)
}

func ToFloat64(ptr *float64) float64 {
if ptr == nil {
return 0
Expand Down
4 changes: 2 additions & 2 deletions providers/aws/resources/aws_applicationautoscaling.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@ func (a *mqlAwsApplicationAutoscaling) getTargets(conn *connection.AwsConnection
"arn": llx.StringData(fmt.Sprintf("arn:aws:application-autoscaling:%s:%s:%s/%s", regionVal, conn.AccountId(), namespace, convert.ToString(target.ResourceId))),
"namespace": llx.StringData(string(target.ServiceNamespace)),
"scalableDimension": llx.StringData(string(target.ScalableDimension)),
"minCapacity": llx.IntData(convert.ToInt64From32(target.MinCapacity)),
"maxCapacity": llx.IntData(convert.ToInt64From32(target.MaxCapacity)),
"minCapacity": llx.IntDataDefault(target.MinCapacity, 0),
"maxCapacity": llx.IntDataDefault(target.MaxCapacity, 0),
"suspendedState": llx.MapData(targetState, types.Any),
})
if err != nil {
Expand Down
8 changes: 4 additions & 4 deletions providers/aws/resources/aws_autoscaling.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,11 @@ func (a *mqlAwsAutoscaling) getGroups(conn *connection.AwsConnection) []*jobpool
"healthCheckType": llx.StringDataPtr(group.HealthCheckType),
"tags": llx.MapData(autoscalingTagsToMap(group.Tags), types.String),
"region": llx.StringData(regionVal),
"minSize": llx.IntData(convert.ToInt64From32(group.MinSize)),
"maxSize": llx.IntData(convert.ToInt64From32(group.MaxSize)),
"defaultCooldown": llx.IntData(convert.ToInt64From32(group.DefaultCooldown)),
"minSize": llx.IntDataDefault(group.MinSize, 0),
"maxSize": llx.IntDataDefault(group.MaxSize, 0),
"defaultCooldown": llx.IntDataDefault(group.DefaultCooldown, 0),
"launchConfigurationName": llx.StringDataPtr(group.LaunchConfigurationName),
"healthCheckGracePeriod": llx.IntData(convert.ToInt64From32(group.HealthCheckGracePeriod)),
"healthCheckGracePeriod": llx.IntDataDefault(group.HealthCheckGracePeriod, 0),
"createdAt": llx.TimeDataPtr(group.CreatedTime),
})
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions providers/aws/resources/aws_cloudfront.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ func (a *mqlAwsCloudfront) distributions() ([]interface{}, error) {
map[string]*llx.RawData{
"domainName": llx.StringDataPtr(o.DomainName),
"id": llx.StringDataPtr(o.Id),
"connectionAttempts": llx.IntData(convert.ToInt64From32(o.ConnectionAttempts)),
"connectionTimeout": llx.IntData(convert.ToInt64From32(o.ConnectionTimeout)),
"connectionAttempts": llx.IntDataDefault(o.ConnectionAttempts, 0),
"connectionTimeout": llx.IntDataDefault(o.ConnectionTimeout, 0),
"originPath": llx.StringDataPtr(o.OriginPath),
"account": llx.StringData(conn.AccountId()),
})
Expand Down
2 changes: 1 addition & 1 deletion providers/aws/resources/aws_cloudwatch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -608,7 +608,7 @@ func (a *mqlAwsCloudwatch) getLogGroups(conn *connection.AwsConnection) []*jobpo
args["arn"] = llx.StringDataPtr(loggroup.Arn)
args["name"] = llx.StringDataPtr(loggroup.LogGroupName)
args["region"] = llx.StringData(regionVal)
args["retentionInDays"] = llx.IntData(convert.ToInt64From32(loggroup.RetentionInDays))
args["retentionInDays"] = llx.IntDataDefault(loggroup.RetentionInDays, 0)

// add kms key if there is one
if loggroup.KmsKeyId != nil {
Expand Down
22 changes: 11 additions & 11 deletions providers/aws/resources/aws_ec2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,16 +158,16 @@ func (a *mqlAwsEc2Networkacl) entries() ([]interface{}, error) {
args := map[string]*llx.RawData{
"egress": llx.BoolData(egress),
"ruleAction": llx.StringData(string(entry.RuleAction)),
"ruleNumber": llx.IntData(convert.ToInt64From32(entry.RuleNumber)),
"ruleNumber": llx.IntDataDefault(entry.RuleNumber, 0),
"cidrBlock": llx.StringDataPtr(entry.CidrBlock),
"ipv6CidrBlock": llx.StringDataPtr(entry.Ipv6CidrBlock),
"id": llx.StringData(entryId),
}
if entry.PortRange != nil {
mqlPortRange, err := CreateResource(a.MqlRuntime, "aws.ec2.networkacl.entry.portrange",
map[string]*llx.RawData{
"from": llx.IntData(convert.ToInt64From32(entry.PortRange.From)),
"to": llx.IntData(convert.ToInt64From32(entry.PortRange.To)),
"from": llx.IntDataDefault(entry.PortRange.From, -1),
"to": llx.IntDataDefault(entry.PortRange.To, -1),
"id": llx.StringData(entryId + "-" + strconv.Itoa(convert.ToIntFrom32(entry.PortRange.From))),
})
if err != nil {
Expand Down Expand Up @@ -269,8 +269,8 @@ func (a *mqlAwsEc2) getSecurityGroups(conn *connection.AwsConnection) []*jobpool
mqlSecurityGroupIpPermission, err := CreateResource(a.MqlRuntime, "aws.ec2.securitygroup.ippermission",
map[string]*llx.RawData{
"id": llx.StringData(convert.ToString(group.GroupId) + "-" + strconv.Itoa(p)),
"fromPort": llx.IntData(convert.ToInt64From32(permission.FromPort)),
"toPort": llx.IntData(convert.ToInt64From32(permission.ToPort)),
"fromPort": llx.IntDataDefault(permission.FromPort, -1),
"toPort": llx.IntDataDefault(permission.ToPort, -1),
"ipProtocol": llx.StringDataPtr(permission.IpProtocol),
"ipRanges": llx.ArrayData(ipRanges, types.Any),
"ipv6Ranges": llx.ArrayData(ipv6Ranges, types.Any),
Expand Down Expand Up @@ -304,8 +304,8 @@ func (a *mqlAwsEc2) getSecurityGroups(conn *connection.AwsConnection) []*jobpool
mqlSecurityGroupIpPermission, err := CreateResource(a.MqlRuntime, "aws.ec2.securitygroup.ippermission",
map[string]*llx.RawData{
"id": llx.StringData(convert.ToString(group.GroupId) + "-" + strconv.Itoa(p) + "-egress"),
"fromPort": llx.IntData(convert.ToInt64From32(permission.FromPort)),
"toPort": llx.IntData(convert.ToInt64From32(permission.ToPort)),
"fromPort": llx.IntDataDefault(permission.FromPort, -1),
"toPort": llx.IntDataDefault(permission.ToPort, -1),
"ipProtocol": llx.StringDataPtr(permission.IpProtocol),
"ipRanges": llx.ArrayData(ipRanges, types.Any),
"ipv6Ranges": llx.ArrayData(ipv6Ranges, types.Any),
Expand Down Expand Up @@ -1062,13 +1062,13 @@ func (a *mqlAwsEc2) getVolumes(conn *connection.AwsConnection) []*jobpool.Job {
"createTime": llx.TimeDataPtr(vol.CreateTime),
"encrypted": llx.BoolDataPtr(vol.Encrypted),
"id": llx.StringDataPtr(vol.VolumeId),
"iops": llx.IntData(convert.ToInt64From32(vol.Iops)),
"iops": llx.IntDataDefault(vol.Iops, 0),
"multiAttachEnabled": llx.BoolDataPtr(vol.MultiAttachEnabled),
"region": llx.StringData(regionVal),
"size": llx.IntData(convert.ToInt64From32(vol.Size)),
"size": llx.IntDataDefault(vol.Size, 0),
"state": llx.StringData(string(vol.State)),
"tags": llx.MapData(Ec2TagsToMap(vol.Tags), types.String),
"throughput": llx.IntData(convert.ToInt64From32(vol.Throughput)),
"throughput": llx.IntDataDefault(vol.Throughput, 0),
"volumeType": llx.StringData(string(vol.VolumeType)),
})
if err != nil {
Expand Down Expand Up @@ -1358,7 +1358,7 @@ func (a *mqlAwsEc2) getSnapshots(conn *connection.AwsConnection) []*jobpool.Job
"state": llx.StringData(string(snapshot.State)),
"tags": llx.MapData(Ec2TagsToMap(snapshot.Tags), types.String),
"volumeId": llx.StringDataPtr(snapshot.VolumeId),
"volumeSize": llx.IntData(convert.ToInt64From32(snapshot.VolumeSize)),
"volumeSize": llx.IntDataDefault(snapshot.VolumeSize, 0),
})
if err != nil {
return nil, err
Expand Down
4 changes: 2 additions & 2 deletions providers/aws/resources/aws_elasticache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,11 +195,11 @@ func (a *mqlAwsElasticache) getCacheClusters(conn *connection.AwsConnection) []*
"logDeliveryConfigurations": llx.ArrayData(logDeliveryConfigurations, types.Any),
"networkType": llx.StringData(string(cluster.NetworkType)),
"notificationConfiguration": llx.StringData(notificationConfiguration),
"numCacheNodes": llx.IntData(convert.ToInt64From32(cluster.NumCacheNodes)),
"numCacheNodes": llx.IntDataDefault(cluster.NumCacheNodes, 0),
"preferredAvailabilityZone": llx.StringDataPtr(cluster.PreferredAvailabilityZone),
"region": llx.StringData(regionVal),
"securityGroups": llx.ArrayData(sgs, types.Resource("aws.ec2.securitygroup")),
"snapshotRetentionLimit": llx.IntData(convert.ToInt64From32(cluster.SnapshotRetentionLimit)),
"snapshotRetentionLimit": llx.IntDataDefault(cluster.SnapshotRetentionLimit, 0),
"transitEncryptionEnabled": llx.BoolData(convert.ToBool(cluster.TransitEncryptionEnabled)),
"transitEncryptionMode": llx.StringData(string(cluster.TransitEncryptionMode)),
})
Expand Down
2 changes: 1 addition & 1 deletion providers/aws/resources/aws_emr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func (a *mqlAwsEmr) getClusters(conn *connection.AwsConnection) []*jobpool.Job {
map[string]*llx.RawData{
"arn": llx.StringDataPtr(cluster.ClusterArn),
"name": llx.StringDataPtr(cluster.Name),
"normalizedInstanceHours": llx.IntData(convert.ToInt64From32(cluster.NormalizedInstanceHours)),
"normalizedInstanceHours": llx.IntDataDefault(cluster.NormalizedInstanceHours, 0),
"outpostArn": llx.StringDataPtr(cluster.OutpostArn),
"status": llx.MapData(jsonStatus, types.String),
"id": llx.StringDataPtr(cluster.Id),
Expand Down
6 changes: 3 additions & 3 deletions providers/aws/resources/aws_iam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ func (a *mqlAwsIam) mqlPolicies(policies []iamtypes.Policy) ([]interface{}, erro
"name": llx.StringDataPtr(policy.PolicyName),
"description": llx.StringDataPtr(policy.Description),
"isAttachable": llx.BoolData(policy.IsAttachable),
"attachmentCount": llx.IntData(convert.ToInt64From32(policy.AttachmentCount)),
"attachmentCount": llx.IntDataDefault(policy.AttachmentCount, 0),
"createDate": llx.TimeDataPtr(policy.CreateDate),
"updateDate": llx.TimeDataPtr(policy.UpdateDate),
})
Expand Down Expand Up @@ -930,10 +930,10 @@ func (a *mqlAwsIamPolicy) attachmentCount() (int64, error) {
arn := a.Arn.Data

policy, err := a.loadPolicy(arn)
if err != nil {
if err != nil || policy.AttachmentCount == nil {
return int64(0), err
}
return convert.ToInt64From32(policy.AttachmentCount), nil
return int64(*policy.AttachmentCount), nil
}

func (a *mqlAwsIamPolicy) createDate() (*time.Time, error) {
Expand Down
7 changes: 3 additions & 4 deletions providers/aws/resources/aws_lambda.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,10 @@ func (a *mqlAwsLambdaFunction) concurrency() (int64, error) {
if err != nil {
return 0, errors.Wrap(err, "could not gather aws lambda function concurrency")
}
if functionConcurrency.ReservedConcurrentExecutions != nil {
return convert.ToInt64From32(functionConcurrency.ReservedConcurrentExecutions), nil
if functionConcurrency.ReservedConcurrentExecutions == nil {
return 0, nil
}

return 0, nil
return int64(*functionConcurrency.ReservedConcurrentExecutions), nil
}

func (a *mqlAwsLambdaFunction) policy() (interface{}, error) {
Expand Down
24 changes: 12 additions & 12 deletions providers/aws/resources/aws_rds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func (a *mqlAwsRds) getDbInstances(conn *connection.AwsConnection) []*jobpool.Jo
"arn": llx.StringDataPtr(dbInstance.DBInstanceArn),
"autoMinorVersionUpgrade": llx.BoolDataPtr(dbInstance.AutoMinorVersionUpgrade),
"availabilityZone": llx.StringDataPtr(dbInstance.AvailabilityZone),
"backupRetentionPeriod": llx.IntData(convert.ToInt64From32(dbInstance.BackupRetentionPeriod)),
"backupRetentionPeriod": llx.IntDataDefault(dbInstance.BackupRetentionPeriod, 0),
"createdTime": llx.TimeDataPtr(dbInstance.InstanceCreateTime),
"dbInstanceClass": llx.StringDataPtr(dbInstance.DBInstanceClass),
"dbInstanceIdentifier": llx.StringDataPtr(dbInstance.DBInstanceIdentifier),
Expand All @@ -104,14 +104,14 @@ func (a *mqlAwsRds) getDbInstances(conn *connection.AwsConnection) []*jobpool.Jo
"id": llx.StringDataPtr(dbInstance.DBInstanceIdentifier),
"multiAZ": llx.BoolDataPtr(dbInstance.MultiAZ),
"name": llx.StringDataPtr(dbInstance.DBName),
"port": llx.IntData(convert.ToInt64From32(dbInstance.DbInstancePort)),
"port": llx.IntDataDefault(dbInstance.DbInstancePort, 0),
"publiclyAccessible": llx.BoolDataPtr(dbInstance.PubliclyAccessible),
"region": llx.StringData(regionVal),
"securityGroups": llx.ArrayData(sgs, types.Resource("aws.ec2.securitygroup")),
"status": llx.StringDataPtr(dbInstance.DBInstanceStatus),
"storageAllocated": llx.IntData(convert.ToInt64From32(dbInstance.AllocatedStorage)),
"storageAllocated": llx.IntDataDefault(dbInstance.AllocatedStorage, 0),
"storageEncrypted": llx.BoolDataPtr(dbInstance.StorageEncrypted),
"storageIops": llx.IntData(convert.ToInt64From32(dbInstance.Iops)),
"storageIops": llx.IntDataDefault(dbInstance.Iops, 0),
"storageType": llx.StringDataPtr(dbInstance.StorageType),
"tags": llx.MapData(rdsTagsToMap(dbInstance.TagList), types.String),
"endpoint": llx.StringDataPtr(dbInstance.Endpoint.Address),
Expand Down Expand Up @@ -261,7 +261,7 @@ func (a *mqlAwsRds) getDbClusters(conn *connection.AwsConnection) []*jobpool.Job
"arn": llx.StringDataPtr(cluster.DBClusterArn),
"autoMinorVersionUpgrade": llx.BoolDataPtr(cluster.AutoMinorVersionUpgrade),
"availabilityZones": llx.ArrayData(stringSliceAZs, types.String),
"backupRetentionPeriod": llx.IntData(convert.ToInt64From32(cluster.BackupRetentionPeriod)),
"backupRetentionPeriod": llx.IntDataDefault(cluster.BackupRetentionPeriod, 0),
"clusterDbInstanceClass": llx.StringDataPtr(cluster.DBClusterInstanceClass),
"createdTime": llx.TimeDataPtr(cluster.ClusterCreateTime),
"deletionProtection": llx.BoolDataPtr(cluster.DeletionProtection),
Expand All @@ -271,14 +271,14 @@ func (a *mqlAwsRds) getDbClusters(conn *connection.AwsConnection) []*jobpool.Job
"id": llx.StringDataPtr(cluster.DBClusterIdentifier),
"members": llx.ArrayData(mqlRdsDbInstances, types.Resource("aws.rds.dbinstance")),
"multiAZ": llx.BoolDataPtr(cluster.MultiAZ),
"port": llx.IntData(convert.ToInt64From32(cluster.Port)),
"port": llx.IntDataDefault(cluster.Port, -1),
"publiclyAccessible": llx.BoolDataPtr(cluster.PubliclyAccessible),
"region": llx.StringData(regionVal),
"securityGroups": llx.ArrayData(sgs, types.Resource("aws.ec2.securitygroup")),
"status": llx.StringDataPtr(cluster.Status),
"storageAllocated": llx.IntData(convert.ToInt64From32(cluster.AllocatedStorage)),
"storageAllocated": llx.IntDataDefault(cluster.AllocatedStorage, 0),
"storageEncrypted": llx.BoolDataPtr(cluster.StorageEncrypted),
"storageIops": llx.IntData(convert.ToInt64From32(cluster.Iops)),
"storageIops": llx.IntDataDefault(cluster.Iops, 0),
"storageType": llx.StringDataPtr(cluster.StorageType),
"tags": llx.MapData(rdsTagsToMap(cluster.TagList), types.String),
})
Expand Down Expand Up @@ -318,14 +318,14 @@ func (a *mqlAwsRdsDbcluster) snapshots() ([]interface{}, error) {
for _, snapshot := range snapshots.DBClusterSnapshots {
mqlDbSnapshot, err := CreateResource(a.MqlRuntime, "aws.rds.snapshot",
map[string]*llx.RawData{
"allocatedStorage": llx.IntData(convert.ToInt64From32(snapshot.AllocatedStorage)),
"allocatedStorage": llx.IntDataDefault(snapshot.AllocatedStorage, 0),
"arn": llx.StringDataPtr(snapshot.DBClusterSnapshotArn),
"createdAt": llx.TimeDataPtr(snapshot.SnapshotCreateTime),
"encrypted": llx.BoolDataPtr(snapshot.StorageEncrypted),
"engine": llx.StringDataPtr(snapshot.Engine),
"engineVersion": llx.StringDataPtr(snapshot.EngineVersion),
"id": llx.StringDataPtr(snapshot.DBClusterSnapshotIdentifier),
"port": llx.IntData(convert.ToInt64From32(snapshot.Port)),
"port": llx.IntDataDefault(snapshot.Port, -1),
"isClusterSnapshot": llx.BoolData(true),
"region": llx.StringData(region),
"status": llx.StringDataPtr(snapshot.Status),
Expand Down Expand Up @@ -363,14 +363,14 @@ func (a *mqlAwsRdsDbinstance) snapshots() ([]interface{}, error) {
for _, snapshot := range snapshots.DBSnapshots {
mqlDbSnapshot, err := CreateResource(a.MqlRuntime, "aws.rds.snapshot",
map[string]*llx.RawData{
"allocatedStorage": llx.IntData(convert.ToInt64From32(snapshot.AllocatedStorage)),
"allocatedStorage": llx.IntDataDefault(snapshot.AllocatedStorage, 0),
"arn": llx.StringDataPtr(snapshot.DBSnapshotArn),
"createdAt": llx.TimeDataPtr(snapshot.SnapshotCreateTime),
"encrypted": llx.BoolDataPtr(snapshot.Encrypted),
"engine": llx.StringDataPtr(snapshot.Engine),
"engineVersion": llx.StringDataPtr(snapshot.EngineVersion),
"id": llx.StringDataPtr(snapshot.DBSnapshotIdentifier),
"port": llx.IntData(convert.ToInt64From32(snapshot.Port)),
"port": llx.IntDataDefault(snapshot.Port, -1),
"isClusterSnapshot": llx.BoolData(false),
"region": llx.StringData(region),
"status": llx.StringDataPtr(snapshot.Status),
Expand Down
Loading

0 comments on commit ae88e62

Please sign in to comment.