Another DNS fix for TIM (#1147)

* create new MEX_DNS

* fix unit test

* fix comment

* unit test

crm-platforms/openstack/openstack-fip-test-heat-expected.yaml

@@ -129,10 +129,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:
@@ -178,10 +185,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:
@@ -217,10 +231,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:
@@ -256,10 +277,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:

crm-platforms/openstack/openstack-heat.go

@@ -512,7 +512,7 @@ func (o *OpenstackPlatform) populateParams(ctx context.Context, VMGroupOrchestra
 				v.CloudConfigParams.ChefParams.ClientKey = key
 			}
 		}
-		userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DNSServers, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16)
+		userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16)
 		if err != nil {
 			return err
 		}

crm-platforms/openstack/openstack-test-heat-expected.yaml

@@ -121,10 +121,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:
@@ -170,10 +177,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:
@@ -209,10 +223,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:
@@ -248,10 +269,17 @@ resources:
                  - echo 'APT::Periodic::Enable "0";' > /etc/apt/apt.conf.d/10cloudinit-disable
                  - apt-get -y purge update-notifier-common ubuntu-release-upgrader-core landscape-common unattended-upgrades
                  - echo "Removed APT and Ubuntu extra packages" | systemd-cat
+                write_files:
+                  - path:  /etc/systemd/resolved.conf
+                    content: |
+                       [Resolve]
+                       DNS=1.1.1.1
+                       FallbackDNS=1.0.0.1
                 chpasswd: { expire: False }
                 ssh_pwauth: False
                 timezone: UTC
                 runcmd:
+                 - systemctl restart systemd-resolved
                  - echo MOBILEDGEX doing ifconfig
                  - ifconfig -a
             metadata:

crm-platforms/openstack/openstack_heat_test.go

@@ -93,7 +93,7 @@ func validateStack(ctx context.Context, t *testing.T, vmgp *vmlayer.VMGroupOrche
 
 	genVMsUserData := make(map[string]string)
 	for _, v := range vmgp.VMs {
-		userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DNSServers, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16)
+		userdata, err := vmlayer.GetVMUserData(v.Name, v.SharedVolume, v.DeploymentManifest, v.Command, &v.CloudConfigParams, reindent16)
 		require.Nil(t, err)
 		genVMsUserData[v.Name] = userdata
 	}

crm-platforms/vsphere/vsphere-orch.go

@@ -221,12 +221,11 @@ func (v *VSpherePlatform) populateOrchestrationParams(ctx context.Context, vmgp
 	for vmidx, vm := range vmgp.VMs {
 		vmHasExternalIp := false
 		vmgp.VMs[vmidx].MetaData = vmlayer.GetVMMetaData(vm.Role, masterIP, vmsphereMetaDataFormatter)
-		userdata, err := vmlayer.GetVMUserData(vm.Name, vm.SharedVolume, vm.DNSServers, vm.DeploymentManifest, vm.Command, &vm.CloudConfigParams, vmsphereUserDataFormatter)
+		userdata, err := vmlayer.GetVMUserData(vm.Name, vm.SharedVolume, vm.DeploymentManifest, vm.Command, &vm.CloudConfigParams, vmsphereUserDataFormatter)
 		if err != nil {
 			return err
 		}
 		vmgp.VMs[vmidx].UserData = userdata
-		vmgp.VMs[vmidx].DNSServers = strings.Join(vmlayer.CloudflareDns, ",")
 		flavormatch := false
 		for _, f := range flavors {
 			if f.Name == vm.FlavorName {
@@ -461,9 +460,13 @@ func (v *VSpherePlatform) CreateVM(ctx context.Context, vm *vmlayer.VMOrchestrat
 		if err != nil {
 			return err
 		}
+		dnsServers := []string{vm.CloudConfigParams.PrimaryDNS}
+		if vm.CloudConfigParams.FallbackDNS != "" {
+			dnsServers = append(dnsServers, vm.CloudConfigParams.FallbackDNS)
+		}
 		custArgs = append(custArgs, []string{"-ip", ip.Address}...)
 		custArgs = append(custArgs, []string{"-netmask", netmask}...)
-		custArgs = append(custArgs, []string{"-dns-server", vm.DNSServers}...)
+		custArgs = append(custArgs, []string{"-dns-server", strings.Join(dnsServers, ",")}...)
 		if ip.Gateway != "" {
 			custArgs = append(custArgs, []string{"-gateway", ip.Gateway}...)
 		}

vmlayer/props.go

@@ -136,8 +136,13 @@ var VMProviderProps = map[string]*edgeproto.PropertyInfo{
 		Description: "Required if infra API endpoint is completely isolated from external network",
 	},
 	"MEX_SUBNET_DNS": {
-		Name:        "Subnet DNS",
-		Description: "Override cloudflare DNS server IP(s) for subnet, e.g. \"8.8.8.8\" or \"1.1.1.1,8.8.8.8\". Set to NONE to use cloud-init settings",
+		Name:        "DNS Override for Subnet",
+		Description: "Set to NONE to use no DNS entry for new subnets.  Otherwise subnet DNS is set to MEX_DNS",
+	},
+	"MEX_DNS": {
+		Name:        "DNS Server(s)",
+		Description: "Override DNS server IP(s), e.g. \"8.8.8.8\" or \"1.1.1.1,8.8.8.8\"",
+		Value:       "1.1.1.1,1.0.0.1",
 	},
 	"MEX_CLOUDLET_FIREWALL_WHITELIST_EGRESS": {
 		Name:        "Cloudlet Firewall Whitelist Egress",
@@ -279,6 +284,11 @@ func (vp *VMProperties) GetCloudletExternalRouter() string {
 	return value
 }
 
+func (vp *VMProperties) GetCloudletDNS() string {
+	value, _ := vp.CommonPf.Properties.GetValue("MEX_DNS")
+	return value
+}
+
 func (vp *VMProperties) GetSubnetDNS() string {
 	value, _ := vp.CommonPf.Properties.GetValue("MEX_SUBNET_DNS")
 	return value

vmlayer/vmconfig.go

@@ -28,10 +28,18 @@ write_files:
   - path: /etc/ssh/trusted-user-ca-keys.pem
     content: {{ .CACert }}
     append: true
+  - path:  /etc/systemd/resolved.conf
+    content: |
+       [Resolve]
+       DNS={{.PrimaryDNS}}
+       {{- if .FallbackDNS}}
+       FallbackDNS={{.FallbackDNS}}
+       {{- end}}
 chpasswd: { expire: False }
 ssh_pwauth: False
 timezone: UTC
 runcmd:
+ - systemctl restart systemd-resolved
  - echo MOBILEDGEX doing ifconfig
  - ifconfig -a`
 
@@ -64,7 +72,7 @@ mounts:
 // VmConfigDataFormatter formats user or meta data to fit into orchestration templates
 type VmConfigDataFormatter func(instring string) string
 
-func GetVMUserData(name string, sharedVolume bool, dnsServers, manifest, command string, cloudConfigParams *VMCloudConfigParams, formatter VmConfigDataFormatter) (string, error) {
+func GetVMUserData(name string, sharedVolume bool, manifest, command string, cloudConfigParams *VMCloudConfigParams, formatter VmConfigDataFormatter) (string, error) {
 	var rc string
 	if manifest != "" {
 		return formatter(manifest), nil
@@ -81,10 +89,6 @@ runcmd:
 			return "", fmt.Errorf("failed to generate template from cloud config params %v, err %v", cloudConfigParams, err)
 		}
 		rc = buf.String()
-
-		if dnsServers != "" {
-			rc += fmt.Sprintf("\n - echo \"dns-nameservers %s\" >> /etc/network/interfaces.d/50-cloud-init.cfg", dnsServers)
-		}
 		if sharedVolume {
 			return formatter(rc + VmCloudConfigShareMount), nil
 		}

vmlayer/vmparams.go

@@ -40,8 +40,6 @@ const (
 	ActionDelete ActionType = "delete"
 )
 
-var CloudflareDns = []string{"1.1.1.1", "1.0.0.1"}
-
 var ClusterTypeKubernetesMasterLabel = "mex-k8s-master"
 var ClusterTypeDockerVMLabel = "mex-docker-vm"
 
@@ -400,6 +398,8 @@ type VMCloudConfigParams struct {
 	ExtraBootCommands []string
 	ChefParams        *chefmgmt.VMChefParams
 	CACert            string
+	PrimaryDNS        string
+	FallbackDNS       string
 }
 
 // VMOrchestrationParams contains all details  that are needed by the orchestator
@@ -419,7 +419,6 @@ type VMOrchestrationParams struct {
 	UserData                string
 	MetaData                string
 	SharedVolume            bool
-	DNSServers              string
 	AuthPublicKey           string
 	DeploymentManifest      string
 	Command                 string
@@ -516,9 +515,11 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont
 	externalNetName := v.VMProperties.GetCloudletExternalNetwork()
 
 	var err error
+	vmDns := strings.Split(v.VMProperties.GetCloudletDNS(), ",")
+	if len(vmDns) > 2 {
+		return nil, fmt.Errorf("Too many DNS servers specified in MEX_DNS")
+	}
 
-	// DNS is applied either at the subnet or VM level
-	vmDns := ""
 	subnetDns := []string{}
 	cloudletSecGrpID := v.VMProperties.GetCloudletSecurityGroupName()
 	if !spec.SkipDefaultSecGrp {
@@ -532,16 +533,9 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont
 	if err != nil {
 		return nil, err
 	}
-	if v.VMProperties.GetSubnetDNS() == NoSubnetDNS {
+	if v.VMProperties.GetSubnetDNS() != NoSubnetDNS {
 		// Contrail workaround, see EDGECLOUD-2420 for details
-		vmDns = strings.Join(CloudflareDns, " ")
-	} else {
-		if v.VMProperties.GetSubnetDNS() != "" {
-			// A value other than NONE or empty means to use the specified servers
-			subnetDns = strings.Split(v.VMProperties.GetSubnetDNS(), ",")
-		} else {
-			subnetDns = CloudflareDns
-		}
+		subnetDns = vmDns
 	}
 
 	vmgp.Netspec, err = ParseNetSpec(ctx, v.VMProperties.GetCloudletNetworkScheme())
@@ -818,6 +812,12 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont
 				vccp.ChefParams = vm.ChefParams
 			}
 			vccp.CACert = vaultSSHCert
+			if len(vmDns) > 0 {
+				vccp.PrimaryDNS = vmDns[0]
+				if len(vmDns) > 1 {
+					vccp.FallbackDNS = vmDns[1]
+				}
+			}
 			// gpu
 			if vm.OptionalResource == "gpu" {
 				gpuCmds := getGpuExtraCommands()
@@ -827,7 +827,6 @@ func (v *VMPlatform) getVMGroupOrchestrationParamsFromGroupSpec(ctx context.Cont
 				Name:                    v.VMProvider.NameSanitize(vm.Name),
 				Id:                      v.VMProvider.IdSanitize(vm.Name),
 				Role:                    role,
-				DNSServers:              vmDns,
 				ImageName:               vm.ImageName,
 				ImageFolder:             vm.ImageFolder,
 				FlavorName:              vm.FlavorName,