diff --git a/helper/schema/schema.go b/helper/schema/schema.go
index df0172fa10..faf8d38563 100644
--- a/helper/schema/schema.go
+++ b/helper/schema/schema.go
@@ -1305,13 +1305,54 @@ func (m schemaMap) diffSet(
 			switch t := schema.Elem.(type) {
 			case *Resource:
 				// This is a complex resource
+
+				// As long as one of the attribute is not marked to be removed, we should unmark the other attributes
+				// that are marked to be removed just because their new value is not specified in config and their
+				// old value is the zero value.
+				isAllSubKNewRemoved := true
+
 				for k2, schema := range t.Schema {
 					subK := fmt.Sprintf("%s.%s.%s", k, code, k2)
 					err := m.diff(subK, schema, diff, d, true)
 					if err != nil {
 						return err
 					}
+
+					if subV, ok := diff.Attributes[subK]; ok && !subV.NewRemoved {
+						isAllSubKNewRemoved = false
+					}
 				}
+				if !isAllSubKNewRemoved {
+					for k2 := range t.Schema {
+						subK := fmt.Sprintf("%s.%s.%s", k, code, k2)
+						if subV, ok := diff.Attributes[subK]; ok && subV.NewRemoved {
+							schemaList := addrToSchema(strings.Split(subK, "."), map[string]*Schema{k: schema})
+							if len(schemaList) == 0 {
+								continue
+							}
+							subSchema := schemaList[len(schemaList)-1]
+
+							var verb string
+							switch subSchema.Type {
+							case TypeBool:
+								verb = "%t"
+							case TypeInt:
+								verb = "%d"
+							case TypeFloat:
+								verb = "%f"
+							case TypeString:
+								verb = "%s"
+							default:
+								return fmt.Errorf("%s: unknown type %#v", k, schema.Type)
+							}
+							if fmt.Sprintf(verb, subSchema.ZeroValue()) == subV.Old {
+								subV.NewRemoved = false
+								subV.New = subV.Old
+							}
+						}
+					}
+				}
+
 			case *Schema:
 				// Copy the schema so that we can set Computed/ForceNew from
 				// the parent schema (the TypeSet).
diff --git a/helper/schema/schema_test.go b/helper/schema/schema_test.go
index 3b5ea8a052..0434412c6a 100644
--- a/helper/schema/schema_test.go
+++ b/helper/schema/schema_test.go
@@ -3161,6 +3161,156 @@ func TestSchemaMap_Diff(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "Set element with unset Optional attributes should not be affected when another element get updated",
+			Schema: map[string]*Schema{
+				"foo": {
+					Type:     TypeSet,
+					Required: true,
+					Elem: &Resource{
+						Schema: map[string]*Schema{
+							"a": {
+								Type:     TypeInt,
+								Required: true,
+							},
+							"b": {
+								Type:     TypeInt,
+								Optional: true,
+							},
+						},
+					},
+					Set: func(v interface{}) int {
+						m := v.(map[string]interface{})
+						return m["a"].(int) + m["b"].(int)
+					},
+				},
+			},
+
+			State: &terraform.InstanceState{
+				Attributes: map[string]string{
+					"foo.#":   "2",
+					"foo.1.a": "1",
+					"foo.1.b": "0",
+					"foo.2.a": "2",
+					"foo.2.b": "0",
+				},
+			},
+
+			Config: map[string]interface{}{
+				"foo": []interface{}{
+					map[string]interface{}{
+						"a": 1,
+					},
+					map[string]interface{}{
+						"a": 3,
+					},
+				},
+			},
+
+			Diff: &terraform.InstanceDiff{
+				Attributes: map[string]*terraform.ResourceAttrDiff{
+					"foo.1.a": {
+						Old: "1",
+						New: "1",
+					},
+					"foo.1.b": {
+						Old: "0",
+						New: "0",
+					},
+					"foo.2.a": {
+						Old:        "2",
+						New:        "0",
+						NewRemoved: true,
+					},
+					"foo.2.b": {
+						Old:        "0",
+						New:        "0",
+						NewRemoved: true,
+					},
+					"foo.3.a": {
+						Old: "",
+						New: "3",
+					},
+					"foo.3.b": {
+						Old: "",
+						New: "",
+					},
+				},
+			},
+
+			Err: false,
+		},
+		{
+			Name: "Set element with unset Optional attributes should not be affected when new element get added",
+			Schema: map[string]*Schema{
+				"foo": {
+					Type:     TypeSet,
+					Required: true,
+					Elem: &Resource{
+						Schema: map[string]*Schema{
+							"a": {
+								Type:     TypeInt,
+								Required: true,
+							},
+							"b": {
+								Type:     TypeInt,
+								Optional: true,
+							},
+						},
+					},
+					Set: func(v interface{}) int {
+						m := v.(map[string]interface{})
+						return m["a"].(int) + m["b"].(int)
+					},
+				},
+			},
+
+			State: &terraform.InstanceState{
+				Attributes: map[string]string{
+					"foo.#":   "1",
+					"foo.1.a": "1",
+					"foo.1.b": "0",
+				},
+			},
+
+			Config: map[string]interface{}{
+				"foo": []interface{}{
+					map[string]interface{}{
+						"a": 1,
+					},
+					map[string]interface{}{
+						"a": 2,
+					},
+				},
+			},
+
+			Diff: &terraform.InstanceDiff{
+				Attributes: map[string]*terraform.ResourceAttrDiff{
+					"foo.#": {
+						Old: "1",
+						New: "2",
+					},
+					"foo.1.a": {
+						Old: "1",
+						New: "1",
+					},
+					"foo.1.b": {
+						Old: "0",
+						New: "0",
+					},
+					"foo.2.a": {
+						Old: "",
+						New: "2",
+					},
+					"foo.2.b": {
+						Old: "",
+						New: "",
+					},
+				},
+			},
+
+			Err: false,
+		},
 	}
 
 	for i, tc := range cases {