From 09b78578bee8ecf0175b8c3af540f9aa4550633b Mon Sep 17 00:00:00 2001 From: Jim Date: Tue, 10 Nov 2020 15:48:45 -0600 Subject: [PATCH] Another DNS fix for TIM (#1147) * create new MEX_DNS * fix unit test * fix comment * unit test --- .../openstack-fip-test-heat-expected.yaml | 28 ++++++++++++++++++ crm-platforms/openstack/openstack-heat.go | 2 +- .../openstack-test-heat-expected.yaml | 28 ++++++++++++++++++ .../openstack/openstack_heat_test.go | 2 +- crm-platforms/vsphere/vsphere-orch.go | 9 ++++-- vmlayer/props.go | 14 +++++++-- vmlayer/vmconfig.go | 14 +++++---- vmlayer/vmparams.go | 29 +++++++++---------- 8 files changed, 99 insertions(+), 27 deletions(-) diff --git a/crm-platforms/openstack/openstack-fip-test-heat-expected.yaml b/crm-platforms/openstack/openstack-fip-test-heat-expected.yaml index 8f21ae31a..00dc7ea8f 100644 --- a/crm-platforms/openstack/openstack-fip-test-heat-expected.yaml +++ b/crm-platforms/openstack/openstack-fip-test-heat-expected.yaml @@ -129,10 +129,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: @@ -178,10 +185,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: @@ -217,10 +231,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: @@ -256,10 +277,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: diff --git a/crm-platforms/openstack/openstack-heat.go b/crm-platforms/openstack/openstack-heat.go index 0ab036a2a..bee84ed3d 100644 --- a/crm-platforms/openstack/openstack-heat.go +++ b/crm-platforms/openstack/openstack-heat.go @@ -512,7 +512,7 @@ func (o *OpenstackPlatform) populateParams(ctx context.Context, VMGroupOrchestra v.CloudConfigParams.ChefParams.ClientKey = key } } - userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DNSServers, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16) + userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16) if err != nil { return err } diff --git a/crm-platforms/openstack/openstack-test-heat-expected.yaml b/crm-platforms/openstack/openstack-test-heat-expected.yaml index ac563ef8e..6846be9e7 100644 --- a/crm-platforms/openstack/openstack-test-heat-expected.yaml +++ b/crm-platforms/openstack/openstack-test-heat-expected.yaml @@ -121,10 +121,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: @@ -170,10 +177,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: @@ -209,10 +223,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: @@ -248,10 +269,17 @@ resources: - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades - echo "Removed APT and Ubuntu extra packages" | systemd-cat + write_files: + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS=1.1.1.1 + FallbackDNS=1.0.0.1 chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a metadata: diff --git a/crm-platforms/openstack/openstack_heat_test.go b/crm-platforms/openstack/openstack_heat_test.go index f9b4bbc40..41f90f329 100644 --- a/crm-platforms/openstack/openstack_heat_test.go +++ b/crm-platforms/openstack/openstack_heat_test.go @@ -93,7 +93,7 @@ func validateStack(ctx context.Context, t *testing.T, vmgp *vmlayer.VMGroupOrche genVMsUserData := make(map[string]string) for _, v := range vmgp.VMs { - userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DNSServers, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16) + userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16) require.Nil(t, err) genVMsUserData[v.Name] = userdata } diff --git a/crm-platforms/vsphere/vsphere-orch.go b/crm-platforms/vsphere/vsphere-orch.go index 498e1b56b..109c500b2 100644 --- a/crm-platforms/vsphere/vsphere-orch.go +++ b/crm-platforms/vsphere/vsphere-orch.go @@ -221,12 +221,11 @@ func (v *VSpherePlatform) populateOrchestrationParams(ctx context.Context, vmgp for vmidx, vm := range vmgp.VMs { vmHasExternalIp := false vmgp.VMs[vmidx].MetaData = vmlayer.GetVMMetaData(vm.Role, masterIP, vmsphereMetaDataFormatter) - userdata, err := vmlayer.GetVMUserData(vm.Name, vm.SharedVolume, vm.DNSServers, vm.DeploymentManifest, vm.Command, &vm.CloudConfigParams, vmsphereUserDataFormatter) + userdata, err := vmlayer.GetVMUserData(vm.Name, vm.SharedVolume, vm.DeploymentManifest, vm.Command, &vm.CloudConfigParams, vmsphereUserDataFormatter) if err != nil { return err } vmgp.VMs[vmidx].UserData = userdata - vmgp.VMs[vmidx].DNSServers = strings.Join(vmlayer.CloudflareDns, ",") flavormatch := false for _, f := range flavors { if f.Name == vm.FlavorName { @@ -461,9 +460,13 @@ func (v *VSpherePlatform) CreateVM(ctx context.Context, vm *vmlayer.VMOrchestrat if err != nil { return err } + dnsServers := []string{vm.CloudConfigParams.PrimaryDNS} + if vm.CloudConfigParams.FallbackDNS != "" { + dnsServers = append(dnsServers, vm.CloudConfigParams.FallbackDNS) + } custArgs = append(custArgs, []string{"-ip", ip.Address}...) custArgs = append(custArgs, []string{"-netmask", netmask}...) - custArgs = append(custArgs, []string{"-dns-server", vm.DNSServers}...) + custArgs = append(custArgs, []string{"-dns-server", strings.Join(dnsServers, ",")}...) if ip.Gateway != "" { custArgs = append(custArgs, []string{"-gateway", ip.Gateway}...) } diff --git a/vmlayer/props.go b/vmlayer/props.go index a73694191..f058a6cc0 100644 --- a/vmlayer/props.go +++ b/vmlayer/props.go @@ -136,8 +136,13 @@ var VMProviderProps = map[string]*edgeproto.PropertyInfo{ Description: "Required if infra API endpoint is completely isolated from external network", }, "MEX_SUBNET_DNS": { - Name: "Subnet DNS", - Description: "Override cloudflare DNS server IP(s) for subnet, e.g. \"8.8.8.8\" or \"1.1.1.1,8.8.8.8\". Set to NONE to use cloud-init settings", + Name: "DNS Override for Subnet", + Description: "Set to NONE to use no DNS entry for new subnets. Otherwise subnet DNS is set to MEX_DNS", + }, + "MEX_DNS": { + Name: "DNS Server(s)", + Description: "Override DNS server IP(s), e.g. \"8.8.8.8\" or \"1.1.1.1,8.8.8.8\"", + Value: "1.1.1.1,1.0.0.1", }, "MEX_CLOUDLET_FIREWALL_WHITELIST_EGRESS": { Name: "Cloudlet Firewall Whitelist Egress", @@ -279,6 +284,11 @@ func (vp *VMProperties) GetCloudletExternalRouter() string { return value } +func (vp *VMProperties) GetCloudletDNS() string { + value, _ := vp.CommonPf.Properties.GetValue("MEX_DNS") + return value +} + func (vp *VMProperties) GetSubnetDNS() string { value, _ := vp.CommonPf.Properties.GetValue("MEX_SUBNET_DNS") return value diff --git a/vmlayer/vmconfig.go b/vmlayer/vmconfig.go index 40e86c266..72343032b 100644 --- a/vmlayer/vmconfig.go +++ b/vmlayer/vmconfig.go @@ -28,10 +28,18 @@ write_files: - path: /etc/ssh/trusted-user-ca-keys.pem content: {{ .CACert }} append: true + - path: /etc/systemd/resolved.conf + content: | + [Resolve] + DNS={{.PrimaryDNS}} + {{- if .FallbackDNS}} + FallbackDNS={{.FallbackDNS}} + {{- end}} chpasswd: { expire: False } ssh_pwauth: False timezone: UTC runcmd: + - systemctl restart systemd-resolved - echo MOBILEDGEX doing ifconfig - ifconfig -a` @@ -64,7 +72,7 @@ mounts: // VmConfigDataFormatter formats user or meta data to fit into orchestration templates type VmConfigDataFormatter func(instring string) string -func GetVMUserData(name string, sharedVolume bool, dnsServers, manifest, command string, cloudConfigParams *VMCloudConfigParams, formatter VmConfigDataFormatter) (string, error) { +func GetVMUserData(name string, sharedVolume bool, manifest, command string, cloudConfigParams *VMCloudConfigParams, formatter VmConfigDataFormatter) (string, error) { var rc string if manifest != "" { return formatter(manifest), nil @@ -81,10 +89,6 @@ runcmd: return "", fmt.Errorf("failed to generate template from cloud config params %v, err %v", cloudConfigParams, err) } rc = buf.String() - - if dnsServers != "" { - rc += fmt.Sprintf("\n - echo \"dns-nameservers %s\" >> /etc/network/interfaces.d/50-cloud-init.cfg", dnsServers) - } if sharedVolume { return formatter(rc + VmCloudConfigShareMount), nil } diff --git a/vmlayer/vmparams.go b/vmlayer/vmparams.go index d51b30c1e..80dc6608b 100644 --- a/vmlayer/vmparams.go +++ b/vmlayer/vmparams.go @@ -40,8 +40,6 @@ const ( ActionDelete ActionType = "delete" ) -var CloudflareDns = []string{"1.1.1.1", "1.0.0.1"} - var ClusterTypeKubernetesMasterLabel = "mex-k8s-master" var ClusterTypeDockerVMLabel = "mex-docker-vm" @@ -400,6 +398,8 @@ type VMCloudConfigParams struct { ExtraBootCommands []string ChefParams *chefmgmt.VMChefParams CACert string + PrimaryDNS string + FallbackDNS string } // VMOrchestrationParams contains all details that are needed by the orchestator @@ -419,7 +419,6 @@ type VMOrchestrationParams struct { UserData string MetaData string SharedVolume bool - DNSServers string AuthPublicKey string DeploymentManifest string Command string @@ -516,9 +515,11 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont externalNetName := v.VMProperties.GetCloudletExternalNetwork() var err error + vmDns := strings.Split(v.VMProperties.GetCloudletDNS(), ",") + if len(vmDns) > 2 { + return nil, fmt.Errorf("Too many DNS servers specified in MEX_DNS") + } - // DNS is applied either at the subnet or VM level - vmDns := "" subnetDns := []string{} cloudletSecGrpID := v.VMProperties.GetCloudletSecurityGroupName() if !spec.SkipDefaultSecGrp { @@ -532,16 +533,9 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont if err != nil { return nil, err } - if v.VMProperties.GetSubnetDNS() == NoSubnetDNS { + if v.VMProperties.GetSubnetDNS() != NoSubnetDNS { // Contrail workaround, see EDGECLOUD-2420 for details - vmDns = strings.Join(CloudflareDns, " ") - } else { - if v.VMProperties.GetSubnetDNS() != "" { - // A value other than NONE or empty means to use the specified servers - subnetDns = strings.Split(v.VMProperties.GetSubnetDNS(), ",") - } else { - subnetDns = CloudflareDns - } + subnetDns = vmDns } vmgp.Netspec, err = ParseNetSpec(ctx, v.VMProperties.GetCloudletNetworkScheme()) @@ -818,6 +812,12 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont vccp.ChefParams = vm.ChefParams } vccp.CACert = vaultSSHCert + if len(vmDns) > 0 { + vccp.PrimaryDNS = vmDns[0] + if len(vmDns) > 1 { + vccp.FallbackDNS = vmDns[1] + } + } // gpu if vm.OptionalResource == "gpu" { gpuCmds := getGpuExtraCommands() @@ -827,7 +827,6 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont Name: v.VMProvider.NameSanitize(vm.Name), Id: v.VMProvider.IdSanitize(vm.Name), Role: role, - DNSServers: vmDns, ImageName: vm.ImageName, ImageFolder: vm.ImageFolder, FlavorName: vm.FlavorName,