map_codec: option to leave default map entry fields out, as prost does

.changes/unreleased/added-20261004-map-entries-omit-defaults.yaml

@@ -0,0 +1,4 @@
+kind: Added
+body: |-
+    **Generated messages can encode map entries as prost does** (#562). The option is `buffa_build::Config::map_entries_omit_defaults`, `map_entries_omit_defaults=true` in `protoc-gen-buffa`, and `CodeGenConfig::map_entries_omit_defaults`; it is off by default. A key or value at its default (zero, `false`, empty, an open enum's 0, a closed enum's first value, or a message value that encodes to nothing) is left out of the entry by the owned, view and lazy view encoders; the entry itself is kept. Decoding is unchanged; as with prost, a `-0.0` float value is left out and so decodes as `0.0`. A message value that equals its default but still encodes a field, such as a `required` field at its default, is written, where prost leaves it out. For maps with several entries to come out in prost's `btree_map` order, also use `map_type(MapRepr::BTreeMap)`. Each crate's types encode as that crate was generated; the well-known types and `DynamicMessage` write both fields, and under `CodecStrategy::Table` messages with a map field keep the unrolled codec, and a `codec_strategy_in` rule that selects the table for such a message by its exact path is an error.
+time: 2026-10-04T21:00:00.000000000+00:00

buffa-build/src/lib.rs

@@ -1490,6 +1490,61 @@ impl Config {
         self
     }
 
+    /// Encode map entries as prost does: leave an entry's key or value out
+    /// when it holds the default (default: false, both are always written, as
+    /// the C++ and Java runtimes write them). Entries themselves are never
+    /// dropped.
+    ///
+    /// For migrations from prost where stored or hashed map encodings must
+    /// not change. Decoding is unchanged: buffa, like every protobuf runtime,
+    /// reads both forms to the same map. One value does not survive the round
+    /// trip: a `-0.0` float value is left out, as prost leaves it out, and
+    /// decodes as `0.0`.
+    ///
+    /// The default is what a missing field decodes to: zero, `false`, empty,
+    /// an open enum's 0 or a closed enum's first value. A message value is
+    /// left out when it encodes to nothing.
+    ///
+    /// # Differences from prost
+    ///
+    /// - prost leaves out a message value equal to its default. A value that
+    ///   equals its default and still encodes a field, such as one with a
+    ///   `required` field at its default or a float field holding `-0.0`, is
+    ///   written here and not by prost.
+    /// - A value of an enum made open with
+    ///   [`open_enums_in`](Self::open_enums_in) is left out at 0, which a
+    ///   missing open enum decodes to, where prost leaves out the enum's
+    ///   first value. The two differ only for an enum whose first value is
+    ///   not 0. protoc rejects that as a map value, so the case can occur
+    ///   only with a descriptor set that another tool built
+    ///   ([`use_buf`](Self::use_buf), [`descriptor_set`](Self::descriptor_set)).
+    /// - The option covers the entries' bytes only. For a map with more than
+    ///   one entry to encode identically, the entries must also come in the
+    ///   same order: use [`map_type`](Self::map_type) with
+    ///   [`MapRepr::BTreeMap`] where prost used `btree_map`.
+    ///
+    /// # Scope
+    ///
+    /// - The option applies to the owned, view and lazy view encoders of the
+    ///   messages generated in this run. Each crate's types encode as that
+    ///   crate was generated.
+    /// - `google.protobuf.Struct`, the well-known type with a map field,
+    ///   comes from `buffa-types` and writes both fields, and so does
+    ///   `DynamicMessage`.
+    /// - A lazy view keeps its singular and repeated message fields undecoded
+    ///   and writes them as they were read, so map entries inside them keep
+    ///   the encoding they arrived with. Every encoder writes entries kept as
+    ///   unknown fields as they were read.
+    /// - With [`CodecStrategy::Table`], messages with a map field keep the
+    ///   unrolled codec, and a
+    ///   [`codec_strategy_in`](Self::codec_strategy_in) rule that selects the
+    ///   table for such a message by its exact path is an error.
+    #[must_use]
+    pub fn map_entries_omit_defaults(mut self, enabled: bool) -> Self {
+        self.codegen_config.map_entries_omit_defaults = enabled;
+        self
+    }
+
     /// Map the matching `map` fields to a custom collection implementing
     /// `buffa::map_codec::MapStorage`, named by its fully-qualified Rust path
     /// (e.g. `"::my_crate::OrderedMap"`). The path must **not** include the

buffa-codegen/src/impl_message.rs

@@ -3209,6 +3209,26 @@ fn map_element_encode_stmt(ty: Type, tag_num: u32, var: &Ident) -> TokenStream {
     quote! { #tag #payload }
 }
 
+/// Whether a view's map entry element bound to `var` is written, that is not
+/// its type's default, which `map_entries_omit_defaults` leaves out of the
+/// entry, as `MapCodec::is_default` decides for owned messages.
+fn map_element_is_written_expr(ty: Type, closed_enum: bool, var: &Ident) -> TokenStream {
+    match ty {
+        Type::TYPE_STRING | Type::TYPE_BYTES => quote! { !#var.is_empty() },
+        // A closed enum's default is its first value, which need not be 0.
+        Type::TYPE_ENUM if closed_enum => quote! {
+            !<::buffa::map_codec::ClosedEnum<_> as ::buffa::map_codec::MapCodec>::is_default(#var)
+        },
+        Type::TYPE_ENUM => quote! { #var.to_i32() != 0 },
+        Type::TYPE_BOOL => quote! { *#var },
+        Type::TYPE_FLOAT | Type::TYPE_DOUBLE => quote! { *#var != 0.0 },
+        Type::TYPE_MESSAGE => {
+            unreachable!("message map values are handled per-phase by callers")
+        }
+        _ => quote! { *#var != 0 },
+    }
+}
+
 fn map_view_compute_size_stmt(
     ctx: &CodeGenContext,
     msg: &DescriptorProto,
@@ -3225,51 +3245,86 @@ fn map_view_compute_size_stmt(
     let (key_fd, val_fd) = find_map_entry_fields(msg, field)?;
     let key_ty = effective_type_in_map_entry(ctx, key_fd, features);
     let val_ty = effective_type_in_map_entry(ctx, val_fd, features);
+    let val_closed_enum = val_ty == Type::TYPE_ENUM
+        && is_closed_enum(&crate::features::resolve_field(ctx, val_fd, features));
     let key_tag_len = tag_encoded_len(1, wire_type_byte(key_ty));
     let val_tag_len = tag_encoded_len(2, wire_type_byte(val_ty));
     let k = format_ident!("k");
     let v = format_ident!("v");
     let key_size = map_element_size_expr(key_ty, &k);
-    let val_size = if val_ty == Type::TYPE_MESSAGE {
+    let omit_defaults = ctx.config.map_entries_omit_defaults;
+    let key_part = if omit_defaults {
+        let written = map_element_is_written_expr(key_ty, false, &k);
+        quote! { (if #written { #key_tag_len + #key_size } else { 0 }) }
+    } else {
+        quote! { #key_tag_len + #key_size }
+    };
+    let val_part = if val_ty == Type::TYPE_MESSAGE && omit_defaults {
+        // Both passes visit every value, so the slots stay in step when an
+        // empty one is left out.
         quote! {
             {
+                let __slot = __cache.reserve();
+                let inner = #v.compute_size(__cache);
+                __cache.set(__slot, inner);
+                if inner == 0 {
+                    0
+                } else {
+                    #val_tag_len
+                        + ::buffa::encoding::varint_len(inner as u64) as u64
+                        + inner as u64
+                }
+            }
+        }
+    } else if val_ty == Type::TYPE_MESSAGE {
+        quote! {
+            #val_tag_len + {
                 let __slot = __cache.reserve();
                 let inner = #v.compute_size(__cache);
                 __cache.set(__slot, inner);
                 ::buffa::encoding::varint_len(inner as u64) as u64 + inner as u64
             }
         }
     } else {
-        map_element_size_expr(val_ty, &v)
+        let val_size = map_element_size_expr(val_ty, &v);
+        if omit_defaults {
+            let written = map_element_is_written_expr(val_ty, val_closed_enum, &v);
+            quote! { (if #written { #val_tag_len + #val_size } else { 0 }) }
+        } else {
+            quote! { #val_tag_len + #val_size }
+        }
     };
     // Both passes iterate `for (k, v) in &self.#ident`, identical to
     // `map_view_write_to_stmt`, so SizeCache slot order matches by construction.
     // When both key and value are fixed-width (no cache slots reserved) the
-    // entry size is constant and we fold to `len() * const`.
-    if map_element_size_is_constant(key_ty) && map_element_size_is_constant(val_ty) {
+    // entry size is constant and we fold to `len() * const`, unless defaults
+    // are left out, which makes it depend on the values.
+    let key_constant = map_element_size_is_constant(key_ty) && !omit_defaults;
+    let val_constant = map_element_size_is_constant(val_ty) && !omit_defaults;
+    if key_constant && val_constant {
         return Ok(quote! {
             {
-                let entry_size: u64 = #key_tag_len + #key_size + #val_tag_len + #val_size;
+                let entry_size: u64 = #key_part + #val_part;
                 size += self.#ident.len() as u64 * (#outer_tag_len
                     + ::buffa::encoding::varint_len(entry_size) as u64
                     + entry_size);
             }
         });
     }
-    let k_bind = if map_element_size_is_constant(key_ty) {
+    let k_bind = if key_constant {
         format_ident!("_{}", k)
     } else {
         k
     };
-    let v_bind = if map_element_size_is_constant(val_ty) {
+    let v_bind = if val_constant {
         format_ident!("_{}", v)
     } else {
         v
     };
     Ok(quote! {
         #[allow(clippy::for_kv_map)]
         for (#k_bind, #v_bind) in &self.#ident {
-            let entry_size: u64 = #key_tag_len + #key_size + #val_tag_len + #val_size;
+            let entry_size: u64 = #key_part + #val_part;
             size += #outer_tag_len
                 + ::buffa::encoding::varint_len(entry_size) as u64
                 + entry_size;
@@ -3292,6 +3347,8 @@ fn map_view_write_to_stmt(
     let (key_fd, val_fd) = find_map_entry_fields(msg, field)?;
     let key_ty = effective_type_in_map_entry(ctx, key_fd, features);
     let val_ty = effective_type_in_map_entry(ctx, val_fd, features);
+    let val_closed_enum = val_ty == Type::TYPE_ENUM
+        && is_closed_enum(&crate::features::resolve_field(ctx, val_fd, features));
     let key_tag_len = tag_encoded_len(1, wire_type_byte(key_ty));
     let val_tag_len = tag_encoded_len(2, wire_type_byte(val_ty));
     let k = format_ident!("k");
@@ -3305,12 +3362,40 @@ fn map_view_write_to_stmt(
     } else {
         (quote! {}, map_element_size_expr(val_ty, &v))
     };
-    let encode_key = map_element_encode_stmt(key_ty, 1, &k);
-    let encode_val = map_element_encode_stmt(val_ty, 2, &v);
+    let mut encode_key = map_element_encode_stmt(key_ty, 1, &k);
+    let mut encode_val = map_element_encode_stmt(val_ty, 2, &v);
+    let mut key_part = quote! { #key_tag_len + #key_size };
+    let mut val_part = quote! { #val_tag_len + #val_size };
+    if ctx.config.map_entries_omit_defaults {
+        let key_written = map_element_is_written_expr(key_ty, false, &k);
+        key_part = quote! { (if #key_written { #key_part } else { 0 }) };
+        encode_key = quote! { if #key_written { #encode_key } };
+        if val_ty == Type::TYPE_MESSAGE {
+            // A value that encodes to nothing loses its tag and length; its
+            // `write_to` still runs, writing nothing but consuming the slots
+            // its `compute_size` reserved.
+            val_part = quote! { (if __v_len == 0 { 0 } else { #val_part }) };
+            let val_tag = quote! {
+                ::buffa::encoding::Tag::new(2, ::buffa::encoding::WireType::LengthDelimited)
+                    .encode(buf);
+            };
+            encode_val = quote! {
+                if __v_len != 0 {
+                    #val_tag
+                    ::buffa::encoding::encode_varint(__v_len as u64, buf);
+                }
+                #v.write_to(__cache, buf);
+            };
+        } else {
+            let val_written = map_element_is_written_expr(val_ty, val_closed_enum, &v);
+            val_part = quote! { (if #val_written { #val_part } else { 0 }) };
+            encode_val = quote! { if #val_written { #encode_val } };
+        }
+    }
     Ok(quote! {
         for (#k, #v) in &self.#ident {
             #val_len_bind
-            let entry_size: u64 = #key_tag_len + #key_size + #val_tag_len + #val_size;
+            let entry_size: u64 = #key_part + #val_part;
             ::buffa::encoding::Tag::new(
                 #field_number,
                 ::buffa::encoding::WireType::LengthDelimited,
@@ -3322,6 +3407,33 @@ fn map_view_write_to_stmt(
     })
 }
 
+/// The `map_codec` functions the owned size and write passes call, picked
+/// together so that the two passes cannot disagree on leaving defaults out.
+struct MapEncoders {
+    field_len: TokenStream,
+    write_field: TokenStream,
+    message_field_len: TokenStream,
+    write_message_field: TokenStream,
+}
+
+fn map_encoders(ctx: &CodeGenContext) -> MapEncoders {
+    if ctx.config.map_entries_omit_defaults {
+        MapEncoders {
+            field_len: quote! { field_len_omitting_defaults },
+            write_field: quote! { write_field_omitting_defaults },
+            message_field_len: quote! { message_field_len_omitting_defaults },
+            write_message_field: quote! { write_message_field_omitting_defaults },
+        }
+    } else {
+        MapEncoders {
+            field_len: quote! { field_len },
+            write_field: quote! { write_field },
+            message_field_len: quote! { message_field_len },
+            write_message_field: quote! { write_message_field },
+        }
+    }
+}
+
 fn map_compute_size_stmt(
     ctx: &CodeGenContext,
     msg: &DescriptorProto,
@@ -3340,17 +3452,22 @@ fn map_compute_size_stmt(
     // Message values are two-pass (SizeCache slot per entry, consumed by
     // `map_write_to_stmt` in identical iteration order — both helpers
     // iterate the same map, so slot order matches by construction).
+    let MapEncoders {
+        field_len,
+        message_field_len,
+        ..
+    } = map_encoders(ctx);
     if m.val_ty == Type::TYPE_MESSAGE {
         return Ok(quote! {
-            size += ::buffa::map_codec::message_field_len::<#key_codec, _, _>(
+            size += ::buffa::map_codec::#message_field_len::<#key_codec, _, _>(
                 &self.#ident,
                 #outer_tag_len,
                 __cache,
             );
         });
     }
     Ok(quote! {
-        size += ::buffa::map_codec::field_len::<#key_codec, #val_codec, _>(
+        size += ::buffa::map_codec::#field_len::<#key_codec, #val_codec, _>(
             &self.#ident,
             #outer_tag_len,
         );
@@ -3372,9 +3489,14 @@ fn map_write_to_stmt(
         val_codec,
         ..
     } = &m;
+    let MapEncoders {
+        write_field,
+        write_message_field,
+        ..
+    } = map_encoders(ctx);
     if m.val_ty == Type::TYPE_MESSAGE {
         return Ok(quote! {
-            ::buffa::map_codec::write_message_field::<#key_codec, _, _>(
+            ::buffa::map_codec::#write_message_field::<#key_codec, _, _>(
                 &self.#ident,
                 #field_number,
                 __cache,
@@ -3383,7 +3505,7 @@ fn map_write_to_stmt(
         });
     }
     Ok(quote! {
-        ::buffa::map_codec::write_field::<#key_codec, #val_codec, _>(
+        ::buffa::map_codec::#write_field::<#key_codec, #val_codec, _>(
             &self.#ident,
             #field_number,
             buf,

buffa-codegen/src/lib.rs

@@ -1099,6 +1099,8 @@ pub enum CodecStrategy {
     /// - a message that uses the `MessageSet` wire format;
     /// - a message with extension ranges, when JSON code is generated and
     ///   unknown fields are preserved;
+    /// - a message with a `map` field, when `map_entries_omit_defaults` is
+    ///   set: the table writes both fields of every entry;
     /// - a message with a field of a non-default string, bytes, or collection
     ///   type, such as `use_bytes_type`, `string_type`, `bytes_type`,
     ///   `repeated_type`, and a custom `map_type` select. `HashMap` and
@@ -1632,6 +1634,27 @@ pub struct CodeGenConfig {
     ///
     /// Defaults to `true`.
     pub generate_with_setters: bool,
+    /// Encode map entries as prost does: leave an entry's key or value out
+    /// when it holds what a missing field decodes to, and a message value when
+    /// it encodes to nothing.
+    ///
+    /// Off, every entry carries both fields, as the C++ and Java runtimes
+    /// write them. Decoding reads either; a `-0.0` float value is left out
+    /// and so decodes as `0.0`. prost leaves out a message value equal to its
+    /// default, so a value that still encodes a field, such as one with a
+    /// `required` field at its default, is written here and not by prost.
+    ///
+    /// Applies to the owned, view and lazy view encoders of the messages
+    /// generated in this run: types generated without it, the well-known
+    /// types and `DynamicMessage` write both fields. With
+    /// [`CodecStrategy::Table`], messages with a map field keep the unrolled
+    /// codec, and a [`codec_strategy_in`](Self::codec_strategy_in) rule that
+    /// selects the table for such a message by its exact path is a
+    /// [`CodeGenError`]. `buffa_build::Config::map_entries_omit_defaults` has
+    /// the details.
+    ///
+    /// Defaults to `false`.
+    pub map_entries_omit_defaults: bool,
     /// Generate `impl Reflectable` for owned message types (bridge mode).
     ///
     /// When enabled, each generated message gets an
@@ -2025,6 +2048,7 @@ impl Default for CodeGenConfig {
             oneof_struct_field_attributes: Vec::new(),
             gate_impls_on_crate_features: false,
             generate_with_setters: true,
+            map_entries_omit_defaults: false,
             generate_reflection: false,
             generate_reflection_vtable: false,
             shared_descriptor_pool: false,

buffa-codegen/src/table_plan.rs

@@ -276,6 +276,13 @@ fn map_field<'a>(
     features: &ResolvedFeatures,
 ) -> Result<TableField<'a>, Ineligible> {
     let name = f.name.as_deref().unwrap_or("");
+    // The table writes both fields of every entry.
+    if ctx.config.map_entries_omit_defaults {
+        return Err(ineligible(
+            "has a map field and map_entries_omit_defaults is set",
+            format!("map_entries_omit_defaults is set and it has map field `{name}`"),
+        ));
+    }
     let field_fqn = format!("{fqn}.{name}");
     if matches!(ctx.map_repr(&field_fqn), crate::MapRepr::Custom(_)) {
         return Err(ineligible(