tmp

Author: rok

parquet/src/column/writer/mod.rs

@@ -1353,19 +1353,25 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> {
         builder: ColumnChunkMetaDataBuilder,
     ) -> ColumnChunkMetaDataBuilder {
         if let Some(encryption_properties) = self.props.file_encryption_properties.as_ref() {
+            let column_crypto_metadata = get_column_crypto_metadata(
+                encryption_properties,
+                &self.descr,
+            );
+
             if encryption_properties.encrypt_footer() {
+                // Encrypted footer: clear statistics from plaintext metadata
                 builder
-                    .set_column_crypto_metadata(get_column_crypto_metadata(
-                        encryption_properties,
-                        &self.descr,
-                    ))
-                    .clear_statistics()    
+                    .set_column_crypto_metadata(column_crypto_metadata)
+                    .clear_statistics()
             } else {
-                builder
-                    .set_column_crypto_metadata(get_column_crypto_metadata(
-                        encryption_properties,
-                        &self.descr,
-                    ))    
+                // Plaintext footer: clear statistics if column is encrypted
+                let builder = builder.set_column_crypto_metadata(column_crypto_metadata.clone());
+                if column_crypto_metadata.is_some() {
+                    // Column is encrypted, strip statistics from plaintext metadata
+                    builder.clear_statistics()
+                } else {
+                    builder
+                }
             }
         } else {
             builder

parquet/src/file/metadata/writer.rs

@@ -632,6 +632,7 @@ impl MetadataObjectWriter {
             Some(file_encryptor) => {
                 let unencrypted_row_groups = row_groups.clone();
                 // TODO: unencrypted_row_groups should not contain statistics for encrypted columns
+
                 let encrypted_row_groups = Self::encrypt_row_groups(row_groups, file_encryptor)?;
                 Ok((encrypted_row_groups, Some(unencrypted_row_groups)))
             }
@@ -783,6 +784,17 @@ impl MetadataObjectWriter {
                 let ciphertext = column_encryptor.encrypt(&buffer, &aad)?;
 
                 column_chunk.encrypted_column_metadata = Some(ciphertext);
+
+                // For plaintext footer mode: strip statistics and encoding_stats from plaintext metadata
+                // These are available in the encrypted_column_metadata for readers with the column key
+                if !file_encryptor.properties().encrypt_footer() {
+                    column_chunk.statistics = None;
+                    // column_chunk.page_encoding_stats = None;
+                    // return Ok(ColumnChunkMetaData {
+                    //     statistics: None,
+                    //     ..column_chunk
+                    // })
+                }
             }
         }
 

parquet/tests/encryption/encryption.rs

@@ -765,15 +765,16 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
 
     let row_group = &metadata.row_groups()[0];
     assert_eq!(row_group.columns().len(), 1);
-    let column_stats = &row_group.columns()[0].statistics().unwrap();
-    assert_eq!(
-        column_stats.min_bytes_opt(),
-        Some(3i32.to_le_bytes().as_slice())
-    );
-    assert_eq!(
-        column_stats.max_bytes_opt(),
-        Some(19i32.to_le_bytes().as_slice())
-    );
+    assert!(&row_group.columns()[0].statistics().is_none());
+    // let column_stats = &row_group.columns()[0].statistics().unwrap();
+    // assert_eq!(
+    //     column_stats.min_bytes_opt(),
+    //     Some(3i32.to_le_bytes().as_slice())
+    // );
+    // assert_eq!(
+    //     column_stats.max_bytes_opt(),
+    //     Some(19i32.to_le_bytes().as_slice())
+    // );
 
     // TODO: statistics shouldn't be available without decryption when footer is plaintext
     //
@@ -786,15 +787,17 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
 
     let row_group = &metadata.row_groups()[0];
     assert_eq!(row_group.columns().len(), 1);
-    let column_stats = &row_group.columns()[0].statistics().unwrap();
-    assert_eq!(
-        column_stats.min_bytes_opt(),
-        Some(3i32.to_le_bytes().as_slice())
-    );
-    assert_eq!(
-        column_stats.max_bytes_opt(),
-        Some(19i32.to_le_bytes().as_slice())
-    );
+    assert!(&row_group.columns()[0].statistics().is_none());
+
+    // let column_stats = &row_group.columns()[0].statistics().unwrap();
+    // assert_eq!(
+    //     column_stats.min_bytes_opt(),
+    //     Some(3i32.to_le_bytes().as_slice())
+    // );
+    // assert_eq!(
+    //     column_stats.max_bytes_opt(),
+    //     Some(19i32.to_le_bytes().as_slice())
+    // );
 
     let builder =
         ParquetRecordBatchReaderBuilder::try_new_with_options(temp_file, options).unwrap();