Skip to content

Commit

Permalink
Add OpenShift Tag to identify Openshift usage and support secondary v…
Browse files Browse the repository at this point in the history 
…nic function
  • Loading branch information
@yutpeng @l-technicore
yutpeng authored and l-technicore committed Nov 27, 2024
1 parent 5adce2f commit 4a88576
Show file tree
Hide file tree
Showing 6 changed files with 353 additions and 7 deletions.
88 changes: 88 additions & 0 deletions pkg/cloudprovider/providers/oci/instances.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ const (
VirtualNodePoolIdAnnotation = "oci.oraclecloud.com/virtual-node-pool-id"
IPv4NodeIPFamilyLabel = "oci.oraclecloud.com/ip-family-ipv4"
IPv6NodeIPFamilyLabel = "oci.oraclecloud.com/ip-family-ipv6"
OpenShiftTagNamesapcePrefix = "openshift-"
OpenShiftBootVolumeType = "boot-volume-type"
OpenShiftBootVolumeISCSI = "ISCSI"
)

var _ cloudprovider.Instances = &CloudProvider{}
Expand Down Expand Up @@ -124,6 +127,59 @@ func (cp *CloudProvider) extractNodeAddresses(ctx context.Context, instanceID st
}
}

OpenShiftTagNamesapce := cp.getOpenShiftTagNamespaceByInstance(ctx, instanceID)

if OpenShiftTagNamesapce != "" {
secondaryVnics, err := cp.client.Compute().GetSecondaryVNICsForInstance(ctx, compartmentID, instanceID)
if err != nil {
return nil, err
}

if secondaryVnics == nil || len(secondaryVnics) == 0 {
return addresses, nil
}
for _, secondaryVnic := range secondaryVnics {
if cp.checkOpenShiftISCSIBootVolumeTagByVnic(ctx, secondaryVnic, OpenShiftTagNamesapce) {
if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PrivateIp != nil && *secondaryVnic.PrivateIp != "" {
ip := net.ParseIP(*secondaryVnic.PrivateIp)
if ip == nil {
return nil, fmt.Errorf("instance has invalid private address: %q", *secondaryVnic.PrivateIp)
}
addresses = append(addresses, api.NodeAddress{Type: api.NodeInternalIP, Address: ip.String()})
}

if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PublicIp != nil && *secondaryVnic.PublicIp != "" {
ip := net.ParseIP(*secondaryVnic.PublicIp)
if ip == nil {
return nil, errors.Errorf("instance has invalid public address: %q", *secondaryVnic.PublicIp)
}
addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()})
}
}
}
nodeIpFamily, err := cp.getNodeIpFamily(instanceID)
if err != nil {
return nil, err
}
if contains(nodeIpFamily, IPv6) {
if vnic.Ipv6Addresses != nil {
for _, ipv6Addresses := range vnic.Ipv6Addresses {
if ipv6Addresses != "" {
ip := net.ParseIP(ipv6Addresses)
if ip == nil {
return nil, errors.Errorf("instance has invalid ipv6 address: %q", vnic.Ipv6Addresses[0])
}
if ip.IsPrivate() {
addresses = append(addresses, api.NodeAddress{Type: api.NodeInternalIP, Address: ip.String()})
} else {
addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()})
}
}
}
}
}
}

// OKE does not support setting DNS since this changes the override hostname we setup to be the ip address.
// Changing this can have wide reaching impact.
//
Expand Down Expand Up @@ -372,3 +428,35 @@ func (cp *CloudProvider) getCompartmentIDByNodeName(nodeName string) (string, er
cp.logger.Debug("CompartmentID annotation is not present")
return "", errors.New("compartmentID annotation missing in the node. Would retry")
}

func (cp *CloudProvider) getOpenShiftTagNamespaceByInstance(ctx context.Context, instanceID string) string {
instance, err := cp.client.Compute().GetInstance(ctx, instanceID)
if err != nil {
return ""
}

if instance.DefinedTags == nil {
return ""
}

for namespace := range instance.DefinedTags {
if strings.HasPrefix(namespace, OpenShiftTagNamesapcePrefix) {
return namespace
}
}
return ""
}

func (cp *CloudProvider) checkOpenShiftISCSIBootVolumeTagByVnic(ctx context.Context, vnic *core.Vnic, namespace string) bool {
if vnic.DefinedTags == nil {
return false
}

if tags, namespaceExists := vnic.DefinedTags[namespace]; namespaceExists {
// Check if the boot volume type key exists and its value is ISCSI
if bootVolume, keyExists := tags[OpenShiftBootVolumeType]; keyExists && bootVolume == OpenShiftBootVolumeISCSI {
return true
}
}
return false
}
196 changes: 196 additions & 0 deletions pkg/cloudprovider/providers/oci/instances_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,24 @@ var (
SubnetId: common.String("ipv6-gua-ipv4-instance"),
Ipv6Addresses: []string{"2001:0db8:85a3:0000:0000:8a2e:0370:7334"},
},
"ocid1.openshift-instance-ipv4": {
IsPrimary: common.Bool(false),
PrivateIp: common.String("10.0.0.1"),
PublicIp: common.String("0.0.0.1"),
HostnameLabel: common.String("openshift-instance"),
SubnetId: common.String("subnetwithdnslabel"),
},
"ocid1.openshift-instance-invalid": {
PrivateIp: common.String("10.0.0."),
HostnameLabel: common.String("openshift-instance-invalid"),
SubnetId: common.String("subnetwithdnslabel"),
},
"ocid1.openshift-instance-ipv6": {
IsPrimary: common.Bool(false),
HostnameLabel: common.String("openshift-instance"),
SubnetId: common.String("ipv6-instance"),
Ipv6Addresses: []string{"2001:0db8:85a3:0000:0000:8a2e:0370:7334"},
},
}

instances = map[string]*core.Instance{
Expand Down Expand Up @@ -205,6 +223,33 @@ var (
Id: common.String("ocid1.ipv6-gua-ipv4-instance"),
CompartmentId: common.String("ipv6-gua-ipv4-instance"),
},
"openshift-instance-ipv4": {
Id: common.String("ocid1.openshift-instance-ipv4"),
CompartmentId: common.String("default"),
DefinedTags: map[string]map[string]interface{}{
"openshift-namespace": {
"role": "compute",
},
},
},
"openshift-instance-invalid": {
Id: common.String("ocid1.openshift-instance-invalid"),
CompartmentId: common.String("default"),
DefinedTags: map[string]map[string]interface{}{
"openshift-namespace": {
"role": "compute",
},
},
},
"openshift-instance-ipv6": {
Id: common.String("ocid1.openshift-instance-ipv6"),
CompartmentId: common.String("default"),
DefinedTags: map[string]map[string]interface{}{
"openshift-namespace": {
"role": "compute",
},
},
},
}
subnets = map[string]*core.Subnet{
"subnetwithdnslabel": {
Expand Down Expand Up @@ -451,6 +496,20 @@ var (
ProviderID: "ocid1.instance-id-ipv6",
},
},
"openshift-instance-id-ipv6": {
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
CompartmentIDAnnotation: "default",
},
Labels: map[string]string{
IPv6NodeIPFamilyLabel: "true",
},
Name: "Node-Ipv6",
},
Spec: v1.NodeSpec{
ProviderID: "ocid1.openshift-instance-ipv6",
},
},
}

podList = map[string]*v1.Pod{
Expand Down Expand Up @@ -873,6 +932,10 @@ func (MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compartm
return instanceVnics[instanceID], nil
}

func (MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
return []*core.Vnic{instanceVnics[instanceID]}, nil
}

func (MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) {
return nil, nil
}
Expand Down Expand Up @@ -1436,6 +1499,29 @@ func TestExtractNodeAddresses(t *testing.T) {
},
err: nil,
},
{
name: "openshift-instance-ipv4",
in: "ocid1.openshift-instance-ipv4",
out: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.0.0.1"},
{Type: v1.NodeExternalIP, Address: "0.0.0.1"},
},
err: nil,
},
{
name: "openshift-instance-invalid",
in: "ocid1.openshift-instance-invalid",
out: nil,
err: errors.New(`instance has invalid private address: "10.0.0."`),
},
{
name: "openshift-instance-ipv6",
in: "ocid1.openshift-instance-ipv6",
out: []v1.NodeAddress{
{Type: v1.NodeExternalIP, Address: "2001:db8:85a3::8a2e:370:7334"},
},
err: nil,
},
}

cp := &CloudProvider{
Expand Down Expand Up @@ -1931,3 +2017,113 @@ func (m mockServiceError) GetOpcRequestID() string {
func (m mockServiceError) Error() string {
return m.Message
}

func TestGetOpenShiftTagNamespaceByInstance(t *testing.T) {
testCases := []struct {
name string
instanceID string
expected string
}{
{
name: "Instance with OpenShift namespace",
instanceID: "openshift-instance-ipv4",
expected: "openshift-namespace",
},
{
name: "Instance without OpenShift namespace",
instanceID: "basic-complete",
expected: "",
},
{
name: "Non-existent instance",
instanceID: "non-existent-instance-id",
expected: "",
},
}

cp := &CloudProvider{
client: MockOCIClient{},
config: &providercfg.Config{CompartmentID: "testCompartment"},
NodeLister: &mockNodeLister{},
instanceCache: &mockInstanceCache{},
}

for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
result := cp.getOpenShiftTagNamespaceByInstance(context.Background(), tt.instanceID)
if !reflect.DeepEqual(result, tt.expected) {
t.Errorf("expected %s, got %s", tt.expected, result)
}
})
}
}

func TestCheckOpenShiftISCSIBootVolumeTagByVnic(t *testing.T) {
testCases := []struct {
name string
vnic *core.Vnic
namespace string
expected bool
}{
{
name: "VNIC with ISCSI boot volume tag",
vnic: &core.Vnic{
DefinedTags: map[string]map[string]interface{}{
"openshift-namespace": {
"boot-volume-type": "ISCSI",
},
},
},
namespace: "openshift-namespace",
expected: true,
},
{
name: "VNIC without ISCSI boot volume tag",
vnic: &core.Vnic{
DefinedTags: map[string]map[string]interface{}{
"openshift-namespace": {
"boot-volume-type": "NVMe",
},
},
},
namespace: "openshift-namespace",
expected: false,
},
{
name: "VNIC with no defined tags",
vnic: &core.Vnic{
DefinedTags: nil,
},
namespace: "openshift-namespace",
expected: false,
},
{
name: "Namespace not found in VNIC tags",
vnic: &core.Vnic{
DefinedTags: map[string]map[string]interface{}{
"another-namespace": {
"bootVolumeType": "ISCSI",
},
},
},
namespace: "openshift-namespace",
expected: false,
},
}

cp := &CloudProvider{
client: MockOCIClient{},
config: &providercfg.Config{CompartmentID: "testCompartment"},
NodeLister: &mockNodeLister{},
instanceCache: &mockInstanceCache{},
}

for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
result := cp.checkOpenShiftISCSIBootVolumeTagByVnic(context.Background(), tt.vnic, tt.namespace)
if !reflect.DeepEqual(result, tt.expected) {
t.Errorf("expected %v, got %v", tt.expected, result)
}
})
}
}
4 changes: 4 additions & 0 deletions pkg/csi/driver/bv_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa
return nil, nil
}

func (MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
return nil, nil
}

func (c *MockComputeClient) ListVnicAttachments(ctx context.Context, compartmentID, instanceID string) ([]core.VnicAttachment, error) {
return nil, nil
}
Expand Down
Loading

0 comments on commit 4a88576

Please sign in to comment.