Expand protocol test coverage for json1.0, query and cbor (#6245)

* Add support for new smithy translated protocol test format

* Various fixes + start adding new query tests

* WIP - adding more new test features

* All query tests passing!

* Add json1.0 tests + fix test with custom host

* Add cbor tests from smithy + add new cborEquals body assertion

* Remove cbor test that java cannot change behavior for

* Add changelog + fix checkstyle

* Fix "/" on query paths

* Fix request nil check

* Revert behavior change in query when request path is set - remove failing protocol tests - avoid behavior changes in this PR

* Minor cleanups

* Add comment to explain content-type in header contains

Author: alextwoods

codegen/src/main/java/software/amazon/awssdk/codegen/model/service/XmlNamespace.java

@@ -18,9 +18,16 @@
 public class XmlNamespace {
 
     private String prefix;
-
     private String uri;
 
+    public XmlNamespace() {
+
+    }
+
+    public XmlNamespace(String uri) {
+        this.uri = uri;
+    }
+
     public String getPrefix() {
         return prefix;
     }

codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java

@@ -302,6 +302,11 @@ public String getEnumValueName(String enumValue) {
         // Special cases
         result = result.replace("textORcsv", "TEXT_OR_CSV");
 
+        // leading digits, add a prefix
+        if (result.matches("^\\d.*")) {
+            result = "VALUE_" + result;
+        }
+
         // Split into words
         result = String.join("_", splitOnWordBoundaries(result));
 

test/protocol-tests-core/pom.xml

@@ -137,6 +137,10 @@
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-cbor</artifactId>
+        </dependency>
     </dependencies>
 
     <!-- Disable spotbugs for this test module to speed up the build. -->

test/protocol-tests-core/src/main/java/software/amazon/awssdk/protocol/asserts/marshalling/CborBodyAssertion.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.protocol.asserts.marshalling;
+
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.DoubleNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import com.github.tomakehurst.wiremock.verification.LoggedRequest;
+import java.util.Base64;
+
+/**
+ * Asserts on the body (expected to be CBOR) of the marshalled request.
+ */
+public class CborBodyAssertion extends MarshallingAssertion {
+    private static final ObjectMapper MAPPER = new ObjectMapper(new CBORFactory());
+
+    private final String cborEquals;
+
+    public CborBodyAssertion(String cborEquals) {
+        this.cborEquals = cborEquals;
+    }
+
+    @Override
+    protected void doAssert(LoggedRequest actual) throws Exception {
+        JsonNode expected = normalizeToDoubles(MAPPER.readTree(Base64.getDecoder().decode(cborEquals)));
+        JsonNode actualJson = normalizeToDoubles(MAPPER.readTree(actual.getBody()));
+        assertEquals(expected, actualJson);
+    }
+
+    /**
+     * CBOR allows serializing floating point numbers to different sizes (eg, float16/32/64).
+     * However, in assertEquals float and double nodes will never be equal so we convert them
+     * all to doubles.
+     */
+    private JsonNode normalizeToDoubles(JsonNode node) {
+        if (node.isFloat() || node.isDouble()) {
+            return DoubleNode.valueOf(node.doubleValue());
+        }
+        if (node.isArray()) {
+            ArrayNode array = MAPPER.createArrayNode();
+            for (JsonNode item : node) {
+                array.add(normalizeToDoubles(item));
+            }
+            return array;
+        }
+        if (node.isObject()) {
+            ObjectNode obj = MAPPER.createObjectNode();
+            node.fields().forEachRemaining(entry -> obj.set(entry.getKey(), normalizeToDoubles(entry.getValue())));
+            return obj;
+        }
+        return node;
+    }
+}

test/protocol-tests-core/src/main/java/software/amazon/awssdk/protocol/asserts/marshalling/HeadersAssertion.java

@@ -21,6 +21,7 @@
 
 import com.github.tomakehurst.wiremock.http.HttpHeaders;
 import com.github.tomakehurst.wiremock.verification.LoggedRequest;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -32,6 +33,7 @@ public class HeadersAssertion extends MarshallingAssertion {
     private Map<String, List<String>> contains;
 
     private List<String> doesNotContain;
+    private List<String> mustContain;
 
     public void setContains(Map<String, List<String>> contains) {
         this.contains = contains;
@@ -41,6 +43,10 @@ public void setDoesNotContain(List<String> doesNotContain) {
         this.doesNotContain = doesNotContain;
     }
 
+    public void setMustContain(List<String> mustContain) {
+        this.mustContain = mustContain;
+    }
+
     @Override
     protected void doAssert(LoggedRequest actual) throws Exception {
         if (contains != null) {
@@ -49,13 +55,20 @@ protected void doAssert(LoggedRequest actual) throws Exception {
         if (doesNotContain != null) {
             assertDoesNotContainHeaders(actual.getHeaders());
         }
+        if (mustContain != null) {
+            assertMustContainHeaders(actual.getHeaders());
+        }
     }
 
     private void assertHeadersContains(HttpHeaders actual) {
         contains.forEach((expectedKey, expectedValues) -> {
             assertTrue(String.format("Header '%s' was expected to be present. Actual headers: %s", expectedKey, actual),
                        actual.getHeader(expectedKey).isPresent());
             List<String> actualValues = actual.getHeader(expectedKey).values();
+            // the Java SDK adds charset to content-type.  This is valid but not included in the protocol tests
+            if (expectedKey.equalsIgnoreCase("Content-Type") && actualValues.size() == 1) {
+                actualValues = Collections.singletonList(actualValues.get(0).replace("; charset=UTF-8", ""));
+            }
             assertEquals(expectedValues, actualValues);
         });
     }
@@ -66,4 +79,11 @@ private void assertDoesNotContainHeaders(HttpHeaders actual) {
                         actual.getHeader(headerName).isPresent());
         });
     }
+
+    private void assertMustContainHeaders(HttpHeaders actual) {
+        mustContain.forEach(headerName -> {
+            assertTrue(String.format("Header '%s' was expected to be present", headerName),
+                        actual.getHeader(headerName).isPresent());
+        });
+    }
 }

test/protocol-tests-core/src/main/java/software/amazon/awssdk/protocol/asserts/marshalling/JsonBodyAssertion.java

@@ -19,6 +19,10 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.IntNode;
+import com.fasterxml.jackson.databind.node.LongNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.github.tomakehurst.wiremock.verification.LoggedRequest;
 
 /**
@@ -37,8 +41,39 @@ public JsonBodyAssertion(String jsonEquals) {
     @Override
     protected void doAssert(LoggedRequest actual) throws Exception {
         JsonNode expected = MAPPER.readTree(jsonEquals);
-        JsonNode actualJson = MAPPER.readTree(actual.getBodyAsString());
+        JsonNode actualJson = convertWholeNumberDoubleToLong(MAPPER.readTree(actual.getBodyAsString()));
         assertEquals(expected, actualJson);
     }
 
+    /**
+     * We serialize some numbers (in particular epoch timestamps) as doubles such as 123.000.
+     * In protocol tests, these values are parsed as longs.  This conversion insures that
+     * 123.000 will equal 123.
+     */
+    public static JsonNode convertWholeNumberDoubleToLong(JsonNode node) {
+        if (node.isDouble()) {
+            double value = node.doubleValue();
+            if (value % 1 == 0) {
+                if (value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE) {
+                    return new IntNode((int) value);
+                }
+                return new LongNode((long) value);
+            }
+        }
+        if (node.isObject()) {
+            ObjectNode obj = (ObjectNode) node;
+            ObjectNode result = obj.objectNode();
+            obj.fieldNames().forEachRemaining(field -> result.set(field, convertWholeNumberDoubleToLong(obj.get(field))));
+            return result;
+        }
+        if (node.isArray()) {
+            ArrayNode array = (ArrayNode) node;
+            ArrayNode result = array.arrayNode();
+            for (JsonNode item : array) {
+                result.add(convertWholeNumberDoubleToLong(item));
+            }
+            return result;
+        }
+        return node;
+    }
 }

test/protocol-tests-core/src/main/java/software/amazon/awssdk/protocol/asserts/marshalling/QueryBodyAssertion.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.protocol.asserts.marshalling;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+import static software.amazon.awssdk.protocol.asserts.marshalling.QueryUtils.parseQueryParams;
+import static software.amazon.awssdk.protocol.asserts.marshalling.QueryUtils.parseQueryParamsFromBody;
+
+import com.github.tomakehurst.wiremock.verification.LoggedRequest;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class QueryBodyAssertion extends MarshallingAssertion  {
+    private final String queryEquals;
+
+    public QueryBodyAssertion(String queryEquals) {
+        this.queryEquals = queryEquals;
+    }
+
+    @Override
+    protected void doAssert(LoggedRequest actual) throws Exception {
+        Map<String, List<String>> expectedParams = parseQueryParamsFromBody(queryEquals);
+        try {
+            Map<String, List<String>> actualParams = parseQueryParams(actual);
+            doAssert(expectedParams, actualParams);
+        } catch (AssertionError error) {
+            // We may send the query params in the body if there is no other content. Try
+            // decoding body as params and rerun the assertions.
+            Map<String, List<String>> actualParams = parseQueryParamsFromBody(
+                actual.getBodyAsString());
+            doAssert(expectedParams, actualParams);
+        }
+    }
+
+    private void doAssert(Map<String, List<String>> expectedParams, Map<String, List<String>> actualParams) {
+        assertThat(actualParams.keySet(), equalTo(expectedParams.keySet()));
+        expectedParams.forEach((key, value) -> assertParamsEqual(actualParams.get(key), value));
+    }
+
+    private void assertParamsEqual(List<String> actual, List<String> expected) {
+        if (expected.stream().allMatch(QueryBodyAssertion::isNumeric)) {
+            assertThat(
+                actual.stream().map(Double::parseDouble).collect(Collectors.toList()),
+                containsInAnyOrder(expected.stream().map(Double::parseDouble).toArray()));
+        } else {
+            assertThat(actual, containsInAnyOrder(expected.toArray()));
+        }
+    }
+
+    public static boolean isNumeric(String str) {
+        if (str == null || str.isEmpty()) {
+            return false;
+        }
+        try {
+            Double.parseDouble(str);
+            return true;
+        } catch (NumberFormatException e) {
+            return false;
+        }
+    }
+
+
+}