tmp

Author: rok

parquet/src/column/writer/mod.rs

@@ -1359,13 +1359,12 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> {
                         encryption_properties,
                         &self.descr,
                     ))
-                    .clear_statistics()    
+                    .clear_statistics()
             } else {
-                builder
-                    .set_column_crypto_metadata(get_column_crypto_metadata(
-                        encryption_properties,
-                        &self.descr,
-                    ))    
+                builder.set_column_crypto_metadata(get_column_crypto_metadata(
+                    encryption_properties,
+                    &self.descr,
+                ))
             }
         } else {
             builder

parquet/src/file/metadata/writer.rs

@@ -631,7 +631,6 @@ impl MetadataObjectWriter {
         match &self.file_encryptor {
             Some(file_encryptor) => {
                 let unencrypted_row_groups = row_groups.clone();
-                // TODO: unencrypted_row_groups should not contain statistics for encrypted columns
                 let encrypted_row_groups = Self::encrypt_row_groups(row_groups, file_encryptor)?;
                 Ok((encrypted_row_groups, Some(unencrypted_row_groups)))
             }

parquet/tests/encryption/encryption.rs

@@ -731,6 +731,7 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
 
     let file_encryption_properties = FileEncryptionProperties::builder(footer_key)
         .with_plaintext_footer(true)
+        .with_column_key("x", column_key.clone())
         .build()
         .unwrap();
 
@@ -740,13 +741,14 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
 
     // Write encrypted data with plaintext footer
     let values = Int32Array::from(vec![8, 3, 4, 19, 5]);
-    let schema = Arc::new(Schema::new(vec![Field::new(
-        "x",
-        values.data_type().clone(),
-        true,
-    )]));
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("x", values.data_type().clone(), true),
+        Field::new("y", values.data_type().clone(), true),
+    ]));
+
     let values = Arc::new(values);
-    let record_batches = vec![RecordBatch::try_new(schema.clone(), vec![values]).unwrap()];
+    let record_batches =
+        vec![RecordBatch::try_new(schema.clone(), vec![values.clone(), values]).unwrap()];
 
     let temp_file = tempfile::tempfile().unwrap();
     let mut writer = ArrowWriter::try_new(&temp_file, schema, Some(props)).unwrap();
@@ -755,7 +757,7 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
     }
     let _file_metadata = writer.close().unwrap();
 
-    // Check column statistics can be read by decrypting
+    // Check column statistics are read given plaintext footer and available decryption properties
     let options =
         ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
     let reader_metadata = ArrowReaderMetadata::load(&temp_file, options.clone()).unwrap();
@@ -764,7 +766,10 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
     assert_eq!(metadata.num_row_groups(), 1);
 
     let row_group = &metadata.row_groups()[0];
-    assert_eq!(row_group.columns().len(), 1);
+    assert_eq!(row_group.columns().len(), 2);
+
+    // Statistics should be available from decrypted data
+    assert!(&row_group.columns()[0].statistics().is_some());
     let column_stats = &row_group.columns()[0].statistics().unwrap();
     assert_eq!(
         column_stats.min_bytes_opt(),
@@ -775,18 +780,19 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
         Some(19i32.to_le_bytes().as_slice())
     );
 
-    // TODO: statistics shouldn't be available without decryption when footer is plaintext
-    //
-    // Check column statistics are not available in plaintext footer
+    // Check column statistics are not read given plaintext footer and not available decryption properties
     let options = ArrowReaderOptions::default();
     let reader_metadata = ArrowReaderMetadata::load(&temp_file, options.clone()).unwrap();
     let metadata = reader_metadata.metadata();
 
     assert_eq!(metadata.num_row_groups(), 1);
 
     let row_group = &metadata.row_groups()[0];
-    assert_eq!(row_group.columns().len(), 1);
-    let column_stats = &row_group.columns()[0].statistics().unwrap();
+    assert_eq!(row_group.columns().len(), 2);
+    assert!(&row_group.columns()[0].statistics().is_none());
+    assert!(&row_group.columns()[1].statistics().is_some());
+
+    let column_stats = &row_group.columns()[1].statistics().unwrap();
     assert_eq!(
         column_stats.min_bytes_opt(),
         Some(3i32.to_le_bytes().as_slice())
@@ -795,14 +801,6 @@ pub fn test_row_group_statistics_plaintext_encrypted_write() {
         column_stats.max_bytes_opt(),
         Some(19i32.to_le_bytes().as_slice())
     );
-
-    let builder =
-        ParquetRecordBatchReaderBuilder::try_new_with_options(temp_file, options).unwrap();
-    let mut record_reader = builder.build().unwrap();
-    assert_eq!(
-        record_reader.next().unwrap().unwrap_err().to_string(),
-        "Parquet argument error: Parquet error: Required field type_ is missing"
-    );
 }
 
 #[test]