sparse codegen

src/aws-cpp-sdk-core/include/aws/core/utils/json/JsonSerializer.h

@@ -94,6 +94,12 @@ namespace Aws
                 JsonValue& WithString(const Aws::String& key, const Aws::String& value);
                 JsonValue& WithString(const char* key, const Aws::String& value);
 
+                /**
+                 * Adds a null value to the top level of this node with key.
+                 */
+                JsonValue& WithNull(const Aws::String& key);
+                JsonValue& WithNull(const char* key);
+
                 /**
                  * Converts the current JSON node to a string.
                  */

src/aws-cpp-sdk-core/source/utils/json/JsonSerializer.cpp

@@ -153,6 +153,23 @@ JsonValue& JsonValue::WithString(const Aws::String& key, const Aws::String& valu
     return WithString(key.c_str(), value);
 }
 
+JsonValue& JsonValue::WithNull(const char* key)
+{
+    if (!m_value)
+    {
+        m_value = cJSON_AS4CPP_CreateObject();
+    }
+
+    const auto val = cJSON_AS4CPP_CreateNull();
+    AddOrReplace(m_value, key, val);
+    return *this;
+}
+
+JsonValue& JsonValue::WithNull(const Aws::String& key)
+{
+    return WithNull(key.c_str());
+}
+
 JsonValue& JsonValue::AsString(const Aws::String& value)
 {
     Destroy();

tests/aws-cpp-sdk-core-tests/utils/json/JsonSerializerTest.cpp

@@ -434,6 +434,39 @@ TEST_F(JsonSerializerTest, TestGetAllObjects)
     ASSERT_EQ(42, all["Key2"].AsInteger());
 }
 
+TEST_F(JsonSerializerTest, TestWithNull)
+{
+    JsonValue value;
+    value.WithString("Key1", "value1");
+    value.WithNull("Key2");
+    value.WithNull(Aws::String("Key3"));
+    value.WithInteger("Key4", 42);
+
+    auto view = value.View();
+
+    ASSERT_TRUE(view.KeyExists("Key2"));
+    ASSERT_FALSE(view.ValueExists("Key2"));
+    ASSERT_TRUE(view.GetObject("Key2").IsNull());
+
+    ASSERT_TRUE(view.KeyExists("Key3"));
+    ASSERT_FALSE(view.ValueExists("Key3"));
+    ASSERT_TRUE(view.GetObject("Key3").IsNull());
+
+    ASSERT_STREQ("value1", view.GetString("Key1").c_str());
+    ASSERT_EQ(42, view.GetInteger("Key4"));
+
+    Aws::String serialized = view.WriteCompact();
+    JsonValue reparsed(serialized);
+    ASSERT_TRUE(reparsed.WasParseSuccessful());
+    auto reparsedView = reparsed.View();
+    ASSERT_TRUE(reparsedView.KeyExists("Key2"));
+    ASSERT_FALSE(reparsedView.ValueExists("Key2"));
+    ASSERT_TRUE(reparsedView.GetObject("Key2").IsNull());
+    ASSERT_TRUE(reparsedView.KeyExists("Key3"));
+    ASSERT_FALSE(reparsedView.ValueExists("Key3"));
+    ASSERT_TRUE(reparsedView.GetObject("Key3").IsNull());
+}
+
 TEST_F(JsonSerializerTest, TestEquality)
 {
     auto input = R"({"AWS" : {

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/domainmodels/c2j/C2jShape.java

@@ -36,4 +36,5 @@ public class C2jShape {
     private boolean sensitive;
     private boolean document;
     private Map<String, Boolean> retryable;
+    private boolean sparse;
 }

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/domainmodels/codegeneration/Shape.java

@@ -63,6 +63,7 @@ public class Shape {
     private boolean overrideStreaming = false;
     private boolean requestCompressionRequired=false;
     private boolean requestCompressionRequiredGzip=false;
+    private boolean sparse=false;
 
     public boolean isMap() {
         return "map".equals(type.toLowerCase());

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/domainmodels/codegeneration/cpp/CppShapeInformation.java

@@ -31,11 +31,15 @@ public class CppShapeInformation {
    private final String requestContentType;
 
    public CppShapeInformation(final Shape shape, final ServiceModel serviceModel) {
+       this(shape, serviceModel, CppViewHelper.computeCppType(shape));
+   }
+
+   public CppShapeInformation(final Shape shape, final ServiceModel serviceModel, final String cppType) {
        this.shape = shape;
        this.serviceModel = serviceModel;
        className = shape.getName();
        exportValue = CppViewHelper.computeExportValue(serviceModel.getMetadata().getClassNamePrefix());
-       cppType = CppViewHelper.computeCppType(shape);
+       this.cppType = cppType;
        headerIncludes = CppViewHelper.computeHeaderIncludes(serviceModel.getMetadata().getProjectName(), shape);
        sourceIncludes = CppViewHelper.computeSourceIncludes(serviceModel.getMetadata().getProjectName(), shape);
        sourceIncludes.removeAll(headerIncludes);

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/domainmodels/codegeneration/cpp/CppViewHelper.java

@@ -225,12 +225,18 @@ else if(shape.isStructure() || shape.isEnum())
 
         else if(shape.isList()) {
             String type = computeCppTypeInternal(shape.getListMember().getShape(), typeMapping);
+            if (shape.isSparse()) {
+                return String.format("Aws::Vector<Aws::Crt::Optional<%s>>", type);
+            }
             return String.format("Aws::Vector<%s>", type);
         }
 
         else if(shape.isMap()) {
             String key = computeCppTypeInternal(shape.getMapKey().getShape(), typeMapping);
             String value = computeCppTypeInternal(shape.getMapValue().getShape(), typeMapping);
+            if (shape.isSparse()) {
+                return String.format("Aws::Map<%s, Aws::Crt::Optional<%s>>", key, value);
+            }
             return String.format("Aws::Map<%s, %s>", key, value);
         }
 
@@ -387,6 +393,9 @@ public static Set<String> computeHeaderIncludes(String projectName, Shape shape)
                     headers.add(formatModelIncludeName(projectName, shapeInList));
                 }
             }
+            if (next.isSparse()) {
+                headers.add("<aws/crt/Optional.h>");
+            }
             if(!next.isPrimitive()) {
                 if (next.isException() && !next.isModeledException()) {
                     // C++ SDK code generator skips generating exceptions that can be expressed using

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/generators/cpp/CborCppClientGenerator.java

@@ -13,6 +13,7 @@
 import com.amazonaws.util.awsclientgenerator.domainmodels.codegeneration.cpp.CppCborViewHelper;
 import com.amazonaws.util.awsclientgenerator.domainmodels.codegeneration.cpp.CppShapeInformation;
 import com.amazonaws.util.awsclientgenerator.domainmodels.codegeneration.cpp.CppViewHelper;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.velocity.Template;
 import org.apache.velocity.VelocityContext;
 
@@ -30,6 +31,18 @@ public CborCppClientGenerator() throws Exception {
         super();
     }
 
+    @Override
+    protected Map<String, CppShapeInformation> buildShapeInformationCache(final ServiceModel serviceModel) {
+        return serviceModel.getShapes().values().stream()
+            .map(shape -> Pair.of(shape.getName(), new CppShapeInformation(shape, serviceModel, CppCborViewHelper.computeCppType(shape))))
+            .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
+    }
+
+    @Override
+    protected Class<?> getViewHelperClass() {
+        return CppCborViewHelper.class;
+    }
+
     @Override
     protected SdkFileEntry generateErrorMarshallerHeaderFile(ServiceModel serviceModel) throws Exception {
         Template template = velocityEngine.getTemplate("/com/amazonaws/util/awsclientgenerator/velocity/cpp/cbor/CborErrorMarshallerHeader.vm", StandardCharsets.UTF_8.name());
@@ -78,7 +91,7 @@ protected SdkFileEntry generateModelHeaderFile(ServiceModel serviceModel, Map.En
             }
 
             context.put("shape", shape);
-            context.put("typeInfo", new CppShapeInformation(shape, serviceModel));
+            context.put("typeInfo", new CppShapeInformation(shape, serviceModel, CppCborViewHelper.computeCppType(shape)));
             context.put("CppViewHelper", CppCborViewHelper.class);
 
             String fileName = String.format("include/aws/%s/model/%s.h", serviceModel.getMetadata().getProjectName(),
@@ -149,7 +162,7 @@ else if (shape.isResult()) {
             }
 
             context.put("shape", shape);
-            context.put("typeInfo", new CppShapeInformation(shape, serviceModel));
+            context.put("typeInfo", new CppShapeInformation(shape, serviceModel, CppCborViewHelper.computeCppType(shape)));
             context.put("CppViewHelper", CppCborViewHelper.class);
 
             String fileName = String.format("source/model/%s.cpp", shapeEntry.getKey());

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/generators/cpp/CppClientGenerator.java

@@ -88,9 +88,7 @@ public SdkFileEntry[] generateSourceFiles(ServiceModel serviceModel) throws Exce
         addEventStreamInitialResponse(serviceModel);
         addRequestIdToResults(serviceModel);
         List<SdkFileEntry> fileList = new ArrayList<>();
-        final Map<String, CppShapeInformation> shapeInformationCache = serviceModel.getShapes().values().stream()
-            .map(shape -> Pair.of(shape.getName(), new CppShapeInformation(shape, serviceModel)))
-            .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
+        final Map<String, CppShapeInformation> shapeInformationCache = buildShapeInformationCache(serviceModel);
         fileList.addAll(generateModelHeaderFiles(serviceModel, shapeInformationCache));
         fileList.addAll(generateModelSourceFiles(serviceModel, shapeInformationCache));
         fileList.add(generateClientHeaderFile(serviceModel));
@@ -256,6 +254,16 @@ private void CheckAndEnableSigV4A(final ServiceModel serviceModel, VelocityConte
         }
     }
 
+    protected Map<String, CppShapeInformation> buildShapeInformationCache(final ServiceModel serviceModel) {
+        return serviceModel.getShapes().values().stream()
+            .map(shape -> Pair.of(shape.getName(), new CppShapeInformation(shape, serviceModel)))
+            .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
+    }
+
+    protected Class<?> getViewHelperClass() {
+        return CppViewHelper.class;
+    }
+
     protected final VelocityContext createContext(final ServiceModel serviceModel) {
         VelocityContext context = new VelocityContext();
         context.put("nl", System.lineSeparator());
@@ -326,7 +334,7 @@ else if (shape.isEvent() && "blob".equals(shape.getEventPayloadType())) {
         }
         context.put("shape", shape);
         context.put("typeInfo", shapeInformationCache.get(shape.getName()));
-        context.put("CppViewHelper", CppViewHelper.class);
+        context.put("CppViewHelper", getViewHelperClass());
 
         String fileName = String.format("include/aws/%s/model/%s.h", serviceModel.getMetadata().getProjectName(),
                 shapeEntry.getKey());

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/transform/C2jModelToGeneratorModelTransformer.java

@@ -439,6 +439,10 @@ Shape convertShapeBasics(C2jShape c2jShape, String shapeName) {
         shape.setEvent(c2jShape.isEvent());
         shape.setException(c2jShape.isException());
         shape.setDocument(c2jShape.isDocument());
+        shape.setSparse(c2jShape.isSparse());
+        if (shape.isSparse() && !shape.isList() && !shape.isMap()) {
+            throw new SourceGenerationFailedException("The sparse trait is only applicable to list and map shapes, but was found on shape: " + shape.getName());
+        }
 
         if (c2jShape.getXmlNamespace() != null) {
             XmlNamespace xmlns = new XmlNamespace();
@@ -850,6 +854,7 @@ Shape cloneShape(Shape shape) {
         cloned.setException(shape.isException());
         cloned.setXmlNamespace(shape.getXmlNamespace());
         cloned.setDocument(shape.isDocument());
+        cloned.setSparse(shape.isSparse());
         return cloned;
     }
 

tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/ModelClassHeaderMembersSource.vm

@@ -42,7 +42,11 @@
   ${spaces}for(const auto& item : $memberVarName)
   ${spaces}{
   ${spaces}  ss << "${locationName}" << item.first;
+#if($member.shape.sparse)
+  ${spaces}  if(item.second.has_value()) { headers.emplace(ss.str(), item.second.value()); }
+#else
   ${spaces}  headers.emplace(ss.str(), item.second);
+#end
   ${spaces}  ss.str("");
   ${spaces}}
 #elseif($member.shape.list)

tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/ModelClassMembersAndInlines.vm

@@ -111,6 +111,24 @@
 #set($mapMember = $member.value.shape)
 #set($keyType = $CppViewHelper.computeCppType($mapMember.mapKey.shape))
 #set($valueType = $CppViewHelper.computeCppType($mapMember.mapValue.shape))
+#if($mapMember.sparse)
+#set($optionalValueType = "Aws::Crt::Optional<${valueType}>")
+#if(!$mapMember.mapKey.shape.primitive && !$mapMember.mapKey.shape.enum && !$mapMember.mapValue.shape.primitive && !$mapMember.mapValue.shape.enum)
+#set($keyTemplType = "${memberKeyWithFirstLetterCapitalized}KeyT")
+#set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}ValueT")
+    template<typename ${keyTemplType} = $keyType, typename ${valueTemplType} = $optionalValueType>
+    ${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${keyTemplType}&& key, ${valueTemplType}&& value) {
+      ${setHasBeenSet}${memberVariableName}.emplace(std::forward<${keyTemplType}>(key), std::forward<${valueTemplType}>(value)); return *this;
+    }
+#else
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${keyType} key, ${valueType} value) {
+      ${setHasBeenSet}${memberVariableName}.emplace(key, value); return *this;
+    }
+#end
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${keyType} key, ${optionalValueType} value) {
+      ${setHasBeenSet}${memberVariableName}.emplace(key, value); return *this;
+    }
+#else
 #if(!$mapMember.mapKey.shape.primitive && !$mapMember.mapKey.shape.enum && !$mapMember.mapValue.shape.primitive && !$mapMember.mapValue.shape.enum)
 #set($keyTemplType = "${memberKeyWithFirstLetterCapitalized}KeyT")
 #set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}ValueT")
@@ -124,16 +142,29 @@
     }
 #end
 #end
+#end
 #if($member.value.shape.list)
 #set($listMember = $member.value.shape)
 #set($valueType = $CppViewHelper.computeCppType($listMember.listMember.shape))
+#if($listMember.sparse)
+#set($optionalValueType = "Aws::Crt::Optional<${valueType}>")
+#if(!$listMember.listMember.shape.primitive && !$listMember.listMember.shape.enum)
+#set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}T")
+    template<typename ${valueTemplType} = $valueType>
+    ${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueTemplType}&& value) { ${setHasBeenSet}${memberVariableName}.emplace_back(std::forward<${valueTemplType}>(value)); return *this; }
+#else
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueType} value)$adderWithCopy
+#end
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${optionalValueType} value)$adderWithCopy
+#else
 #if(!$listMember.listMember.shape.primitive && !$listMember.listMember.shape.enum)
 #set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}T")
     template<typename ${valueTemplType} = $valueType>
     ${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueTemplType}&& value) { ${setHasBeenSet}${memberVariableName}.emplace_back(std::forward<${valueTemplType}>(value)); return *this; }
 #else
     ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueType} value)$adderWithCopy
 #end
+#end
 #end
     ///@}
 #end##if(!($CppViewHelper.isStreamingPayloadMember($shape, $member.key) && $shape.request))