Fix fabric8io#2510: Yaml containing aliases rejected due to FasterXML…

… bug + Fallback to SnakeYAML while deserializing Kubernetes YAML manifests + Refactored SerializationUtils#dumpAsYaml(...) to use SnakeYAML instead of Jackson + Moved `SerializationUtils` to `io.fabric8.kubernetes.client.utils`
rohanKanojia · Dec 2, 2020 · 8bc94e1 · 8bc94e1
1 parent 648117e
commit 8bc94e1
Show file tree

Hide file tree

Showing 13 changed files with 325 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,13 +4,17 @@
 
 #### Bugs
 * Fix #2519: Generated schemas contains a valid meta-schema URI reference (`http://json-schema.org/draft-05/schema#`)
+* FIx #2510 : Yaml containing aliases rejected due to FasterXML bug
 
 #### Improvements
 
 #### Dependency Upgrade
 
 #### New Features
 
+#### Breaking Changes
+- `io.fabric8.kubernetes.client.internal.SerializationUtils` moved to `io.fabric8.kubernetes.client.utils.SerializationUtils`
+
 ### 5.0.0-alpha-3 (2020-11-24)
 
 #### Improvements

diff --git a/...lient/src/main/java/io/fabric8/kubernetes/client/dsl/internal/apps/v1/RollingUpdater.java b/...lient/src/main/java/io/fabric8/kubernetes/client/dsl/internal/apps/v1/RollingUpdater.java
@@ -48,7 +48,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static io.fabric8.kubernetes.client.internal.SerializationUtils.dumpWithoutRuntimeStateAsYaml;
+import static io.fabric8.kubernetes.client.utils.SerializationUtils.dumpWithoutRuntimeStateAsYaml;
 
 public abstract class RollingUpdater<T extends HasMetadata, L> {
   public static final String DEPLOYMENT_KEY = "deployment";

diff --git a/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/IOHelpers.java b/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/IOHelpers.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -29,6 +30,7 @@
 /**
  */
 public class IOHelpers {
+    private IOHelpers() { }
 
     public static String readFully(InputStream in) throws IOException {
         Reader r = new BufferedReader(new InputStreamReader(in));
@@ -75,4 +77,20 @@ public static String convertYamlToJson(String yaml) throws IOException {
     return jsonWriter.writeValueAsString(obj);
   }
 
+  /**
+   * Helper method to copy data from InputStream to ByteArrayOutputStream so that stream contents
+   * could be re-used.
+   *
+   * @param inputStream source InputStream containing data
+   * @param byteArrayOutputStream output ByteArrayOutputStream which will store InputStream contents
+   * @throws IOException in case of failure while reading stream
+   */
+  public static void copy(InputStream inputStream, ByteArrayOutputStream byteArrayOutputStream) throws IOException {
+    int nRead;
+    byte[] data = new byte[1024];
+    while ((nRead = inputStream.read(data, 0, data.length)) != -1) {
+      byteArrayOutputStream.write(data, 0, nRead);
+    }
+  }
+
 }
diff --git a/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/Serialization.java b/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/Serialization.java
@@ -22,6 +22,8 @@
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import io.fabric8.kubernetes.api.model.KubernetesResource;
 import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.internal.KubernetesDeserializer;
+import org.yaml.snakeyaml.Yaml;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
@@ -46,6 +48,7 @@ private Serialization() { }
     JSON_MAPPER.registerModule(new JavaTimeModule());
   }
   private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory());
+  private static final Yaml SNAKE_YAML_MAPPER = new Yaml();
   private static final String DOCUMENT_DELIMITER = "---";
 
   public static ObjectMapper jsonMapper() {
@@ -56,6 +59,10 @@ public static ObjectMapper yamlMapper() {
     return YAML_MAPPER;
   }
 
+  public static Yaml snakeYamlMapper() {
+    return SNAKE_YAML_MAPPER;
+  }
+
   public static <T> String asJson(T object) {
     try {
       return JSON_MAPPER.writeValueAsString(object);
@@ -132,7 +139,7 @@ public static <T> T unmarshal(InputStream is, ObjectMapper mapper, Map<String, S
       bis.reset();
 
       if (intch != '{') {
-        mapper = YAML_MAPPER;
+        return unmarshalYaml(null, bis);
       }
       return mapper.readerFor(KubernetesResource.class).readValue(bis);
     } catch (IOException e) {
@@ -237,7 +244,7 @@ public static <T> T unmarshal(InputStream is, TypeReference<T> type, Map<String,
 
       ObjectMapper mapper = JSON_MAPPER;
       if (intch != '{') {
-        mapper = YAML_MAPPER;
+        return unmarshalYaml(type, bis);
       }
       return mapper.readValue(bis, type);
     } catch (IOException e) {
@@ -294,4 +301,32 @@ private static String readSpecFileFromInputStream(InputStream inputStream) {
       throw new RuntimeException("Unable to read InputStream." + e);
     }
   }
+
+  private static <T> T unmarshalYaml(TypeReference<T> type, BufferedInputStream bis) throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    IOHelpers.copy(bis, baos);
+    byte[] bytes = baos.toByteArray();
+    try {
+      ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+      if (type == null) {
+        return yamlMapper().readerFor(KubernetesResource.class).readValue(bais);
+      }
+      return yamlMapper().readValue(bais, type);
+    } catch (IOException e) {
+      ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+      return unmarshalUsingSnakeYAML(bais);
+    }
+  }
+
+  private static <T> T unmarshalUsingSnakeYAML(ByteArrayInputStream bis) {
+      Map<String, Object> yamlAsMap = snakeYamlMapper().load(bis);
+      String apiVersion = yamlAsMap.get("apiVersion").toString();
+      String kind = yamlAsMap.get("kind").toString();
+      bis.reset();
+      return (T) unmarshalUsingSnakeYAML(bis, KubernetesDeserializer.getTypeFromName(apiVersion, kind));
+  }
+
+  private static <T> T unmarshalUsingSnakeYAML(ByteArrayInputStream bis, Class<T> type) {
+      return snakeYamlMapper().loadAs(bis, type);
+  }
 }
diff --git a/...s/client/internal/SerializationUtils.java → ...etes/client/utils/SerializationUtils.java b/...s/client/internal/SerializationUtils.java → ...etes/client/utils/SerializationUtils.java
@@ -13,30 +13,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.fabric8.kubernetes.client.internal;
+package io.fabric8.kubernetes.client.utils;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import io.fabric8.kubernetes.api.model.HasMetadata;
-import io.fabric8.kubernetes.api.model.KubernetesResource;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.ReplicationController;
 import io.fabric8.kubernetes.client.internal.serializationmixins.ObjectMetaMixIn;
 import io.fabric8.kubernetes.client.internal.serializationmixins.ReplicationControllerMixIn;
-import io.fabric8.kubernetes.client.utils.Serialization;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.introspector.Property;
+import org.yaml.snakeyaml.nodes.CollectionNode;
+import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.nodes.NodeTuple;
+import org.yaml.snakeyaml.nodes.SequenceNode;
+import org.yaml.snakeyaml.nodes.Tag;
+import org.yaml.snakeyaml.representer.Representer;
 
 public class SerializationUtils {
+  private SerializationUtils() { }
 
   private static ObjectMapper mapper;
 
@@ -58,12 +56,46 @@ public static ObjectMapper getMapper() {
     return mapper;
   }
 
-  public static String dumpAsYaml(HasMetadata obj) throws JsonProcessingException {
-    return getMapper().writeValueAsString(obj);
+  public static Yaml snakeYamlMapper(Representer representer, DumperOptions options) {
+    return new Yaml(representer, options);
+  }
+
+  /**
+   * Dump any Kubernetes resource which extends {@link HasMetadata} object as a YAML
+   *
+   * @param obj {@link HasMetadata} Kubernetes resource
+   * @return String as YAML output
+   */
+  public static String dumpAsYaml(HasMetadata obj) {
+    Representer representer = new SkipNullAndEmptyNodeRepresenter();
+    representer.addClassTag(obj.getClass(), Tag.MAP);
+    DumperOptions options = new DumperOptions();
+    options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+
+    return snakeYamlMapper(representer, options).dump(obj);
   }
 
   public static String dumpWithoutRuntimeStateAsYaml(HasMetadata obj) throws JsonProcessingException {
     return getStatelessMapper().writeValueAsString(obj);
   }
 
+  private static class SkipNullAndEmptyNodeRepresenter extends Representer {
+    @Override
+    protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) {
+      if (propertyValue == null) {
+        return null;
+      }
+      NodeTuple tuple = super
+        .representJavaBeanProperty(javaBean, property, propertyValue, customTag);
+      Node valueNode = tuple.getValueNode();
+      if (valueNode instanceof CollectionNode && Tag.SEQ.equals(valueNode.getTag())) {
+        SequenceNode seq = (SequenceNode) valueNode;
+        if (seq.getValue().isEmpty()) {
+          return null;
+        }
+      }
+      return tuple;
+    }
+  }
+
 }
diff --git a/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/utils/IOHelpersTest.java b/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/utils/IOHelpersTest.java
@@ -0,0 +1,79 @@
+/**
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.fabric8.kubernetes.client.utils;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class IOHelpersTest {
+  @Test
+  void testCopyInputStream() throws IOException {
+    // Given
+    String inputStr = "testStr";
+    InputStream is = new ByteArrayInputStream(inputStr.getBytes());
+    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(128);
+
+    // When
+    IOHelpers.copy(is, byteArrayOutputStream);
+
+    // Then
+    assertEquals(inputStr, new String(byteArrayOutputStream.toByteArray()));
+  }
+
+  @Test
+  void testConvertYamlToJson() throws IOException {
+    // Given
+    String yaml = "apiVersion: apps/v1\n" +
+      "kind: Deployment";
+
+    // When
+    String result = IOHelpers.convertYamlToJson(yaml);
+
+    // Then
+    assertNotNull(result);
+    assertEquals("{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\"}", result);
+  }
+
+  @Test
+  void testIsJsonValid() {
+    assertTrue(IOHelpers.isJSONValid("{\"a\": 3}"));
+    assertFalse(IOHelpers.isJSONValid("{\"a\": 3]}"));
+    assertFalse(IOHelpers.isJSONValid("apiVersion: v1\nkind: Pod"));
+  }
+
+  @Test
+  void testReadFully() throws IOException {
+    // Given
+    String inputStr = "testStr";
+    InputStream is = new ByteArrayInputStream(inputStr.getBytes());
+
+    // When
+    String result = IOHelpers.readFully(is);
+
+    // Then
+    assertNotNull(result);
+    assertEquals(inputStr, result);
+  }
+}
diff --git a/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/utils/SerializationTest.java b/kubernetes-client/src/test/java/io/fabric8/kubernetes/client/utils/SerializationTest.java
@@ -17,17 +17,19 @@
 
 import io.fabric8.kubernetes.api.model.KubernetesList;
 import io.fabric8.kubernetes.api.model.KubernetesResource;
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.api.model.Quantity;
 import io.fabric8.kubernetes.api.model.Service;
 import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.CustomResourceDefinition;
 import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.JSONSchemaProps;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
 
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.Map;
@@ -43,8 +45,6 @@
 import com.fasterxml.jackson.databind.node.TextNode;
 
 public class SerializationTest {
-
-
   @Test
   void unmarshalCRDWithSchema() throws Exception {
     final String input = readYamlToString("/test-crd-schema.yml");
@@ -139,4 +139,28 @@ void containsMultipleDocumentsWithSingleDocumentAndLinuxLineEnds() {
     // Then
     assertFalse(result);
   }
+
+  @Test
+  void testSerializeYamlWithAlias() {
+    // Given
+    InputStream fileInputStream = getClass().getResourceAsStream("/test-pod-manifest-with-aliases.yml");
+
+    // When
+    Pod pod = Serialization.unmarshal(fileInputStream);
+
+    // Then
+    assertNotNull(pod);
+    assertEquals("test-pod-with-alias", pod.getMetadata().getName());
+    assertEquals("build", pod.getSpec().getNodeSelector().get("workload"));
+    assertEquals(1, pod.getSpec().getTolerations().size());
+    assertEquals(1000, pod.getSpec().getSecurityContext().getRunAsGroup().intValue());
+    assertEquals(1000, pod.getSpec().getSecurityContext().getRunAsUser().intValue());
+    assertEquals(2, pod.getSpec().getContainers().size());
+    assertEquals("ubuntu", pod.getSpec().getContainers().get(0).getName());
+    assertEquals("ubuntu:bionic", pod.getSpec().getContainers().get(0).getImage());
+    assertEquals(new Quantity("100m"), pod.getSpec().getContainers().get(0).getResources().getRequests().get("cpu"));
+    assertEquals("python3", pod.getSpec().getContainers().get(1).getName());
+    assertEquals("python:3.7", pod.getSpec().getContainers().get(1).getImage());
+    assertEquals(new Quantity("100m"), pod.getSpec().getContainers().get(1).getResources().getRequests().get("cpu"));
+  }
 }