Add Parquet Modular encryption support (write) #7111

rok · 2025-02-10T22:02:07Z

Which issue does this PR close?

This PR is based on branch and an internal patch and /pull/6637 and aims to provide basic modular encryption support. Partially closes #3511. We decided to split encryption work into a separate PR.

Rationale for this change

See #3511.

What changes are included in this PR?

TBD

Are there any user-facing changes?

Several new classes and method parameters are introduced. If project is compiled without encryption flag changes are not breaking. If encryption flag is on some methods and constructors (e.g. ParquetMetaData::new) will require new parameters which would be a breaking change.

diff --git c/parquet/tests/arrow_reader/encryption_util.rs i/parquet/tests/arrow_reader/encryption_util.rs index 627ac666e..604cd3034 100644 --- c/parquet/tests/arrow_reader/encryption_util.rs +++ i/parquet/tests/arrow_reader/encryption_util.rs @@ -15,13 +15,17 @@ // specific language governing permissions and limitations // under the License. +use crate::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, +}; use arrow_array::cast::AsArray; use arrow_array::{types, RecordBatch}; -use futures::TryStreamExt; + use std::fs::File; use parquet::file::metadata::ParquetMetaData; use crate::arrow::ArrowWriter; use crate::encryption::encrypt::FileEncryptionProperties; +use crate::encryption::decrypt::FileDecryptionProperties; use crate::file::properties::WriterProperties; /// Tests reading an encrypted file from the parquet-testing repository @@ -109,18 +113,24 @@ pub fn verify_encryption_test_data(record_batches: Vec<RecordBatch>, metadata: & } #[cfg(feature = "encryption")] -pub fn read_and_roundtrip_to_encrypted_file(path: &str, decryption_properties: FileDecryptionProperties, encryption_properties: FileEncryptionProperties) { +pub fn read_and_roundtrip_to_encrypted_file( + path: &str, + decryption_properties: FileDecryptionProperties, + encryption_properties: FileEncryptionProperties, +) { let temp_file = tempfile::tempfile().unwrap(); // read example data let file = File::open(path).unwrap(); - let options = - ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties.clone()); + let options = ArrowReaderOptions::default() + .with_file_decryption_properties(decryption_properties.clone()); let metadata = ArrowReaderMetadata::load(&file, options.clone()).unwrap(); let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); let batch_reader = builder.build().unwrap(); - let batches = batch_reader.collect::<crate::errors::Result<Vec<RecordBatch>, _>>().unwrap(); + let batches = batch_reader + .collect::<crate::errors::Result<Vec<RecordBatch>, _>>() + .unwrap(); // write example data let props = WriterProperties::builder()

diff --git c/parquet/README.md i/parquet/README.md index 9245664b4..8fc72bfbc 100644 --- c/parquet/README.md +++ i/parquet/README.md @@ -84,7 +84,7 @@ The `parquet` crate provides the following features which may be enabled in your - [ ] Row record writer - [x] Arrow record writer - [x] Async support - - [ ] Encrypted files + - [x] Encrypted files - [x] Predicate pushdown - [x] Parquet format 4.0.0 support diff --git c/parquet/examples/read_with_rowgroup.rs i/parquet/examples/read_with_rowgroup.rs index 44d255961..8cccc7fe1 100644 --- c/parquet/examples/read_with_rowgroup.rs +++ i/parquet/examples/read_with_rowgroup.rs @@ -35,12 +35,7 @@ async fn main() -> Result<()> { let mut file = File::open(&path).await.unwrap(); // The metadata could be cached in other places, this example only shows how to read - let metadata = file - .get_metadata( - #[cfg(feature = "encryption")] - None, - ) - .await?; + let metadata = file.get_metadata().await?; for rg in metadata.row_groups() { let mut rowgroup = InMemoryRowGroup::create(rg.clone(), ProjectionMask::all()); @@ -126,8 +121,6 @@ impl RowGroups for InMemoryRowGroup { self.metadata.column(i), self.num_rows(), None, - #[cfg(feature = "encryption")] - None, )?); Ok(Box::new(ColumnChunkIterator { diff --git c/parquet/src/arrow/arrow_writer/mod.rs i/parquet/src/arrow/arrow_writer/mod.rs index 0e087b1b2..527e79e61 100644 --- c/parquet/src/arrow/arrow_writer/mod.rs +++ i/parquet/src/arrow/arrow_writer/mod.rs @@ -1311,9 +1311,7 @@ mod tests { use crate::basic::Encoding; use crate::data_type::AsBytes; #[cfg(feature = "encryption")] - use crate::encryption::{ - decryption::FileDecryptionProperties, encrypt::FileEncryptionProperties, - }; + use crate::encryption::{decrypt::FileDecryptionProperties, encrypt::FileEncryptionProperties}; use crate::file::metadata::ParquetMetaData; use crate::file::page_index::index::Index; use crate::file::page_index::index_reader::read_offset_indexes; @@ -3812,8 +3810,8 @@ mod tests { let column_2_key = "1234567890123451".as_bytes(); let decryption_properties = FileDecryptionProperties::builder(footer_key.to_vec()) - .with_column_key("double_field".as_bytes().to_vec(), column_1_key.to_vec()) - .with_column_key("float_field".as_bytes().to_vec(), column_2_key.to_vec()) + .with_column_key("double_field", column_1_key.to_vec()) + .with_column_key("float_field", column_2_key.to_vec()) .build() .unwrap(); diff --git c/parquet/src/arrow/async_reader/metadata.rs i/parquet/src/arrow/async_reader/metadata.rs index 4f41cd6ce..71d2e57dd 100644 --- c/parquet/src/arrow/async_reader/metadata.rs +++ i/parquet/src/arrow/async_reader/metadata.rs @@ -128,26 +128,13 @@ impl<F: MetadataFetch> MetadataLoader<F> { let (metadata, remainder) = if length > suffix_len - FOOTER_SIZE { let metadata_start = file_size - length - FOOTER_SIZE; let meta = fetch.fetch(metadata_start..file_size - FOOTER_SIZE).await?; - ( - ParquetMetaDataReader::decode_metadata( - &meta, - footer.is_encrypted_footer(), - #[cfg(feature = "encryption")] - None, - )?, - None, - ) + (ParquetMetaDataReader::decode_metadata(&meta)?, None) } else { let metadata_start = file_size - length - FOOTER_SIZE - footer_start; let slice = &suffix[metadata_start..suffix_len - FOOTER_SIZE]; ( - ParquetMetaDataReader::decode_metadata( - slice, - footer.is_encrypted_footer(), - #[cfg(feature = "encryption")] - None, - )?, + ParquetMetaDataReader::decode_metadata(slice)?, Some((footer_start, suffix.slice(..metadata_start))), ) }; diff --git c/parquet/src/column/writer/mod.rs i/parquet/src/column/writer/mod.rs index 32139e6cd..91259ab66 100644 --- c/parquet/src/column/writer/mod.rs +++ i/parquet/src/column/writer/mod.rs @@ -1542,9 +1542,7 @@ mod tests { reader::{get_column_reader, get_typed_column_reader, ColumnReaderImpl}, }; #[cfg(feature = "encryption")] - use crate::encryption::{ - decryption::FileDecryptionProperties, encrypt::FileEncryptionProperties, - }; + use crate::encryption::{decrypt::FileDecryptionProperties, encrypt::FileEncryptionProperties}; use crate::file::writer::TrackedWrite; use crate::file::{ properties::ReaderProperties, reader::SerializedPageReader, writer::SerializedPageWriter, @@ -2126,8 +2124,6 @@ mod tests { r.rows_written as usize, None, Arc::new(props), - #[cfg(feature = "encryption")] - None, ) .unwrap(); @@ -2180,8 +2176,6 @@ mod tests { r.rows_written as usize, None, Arc::new(props), - #[cfg(feature = "encryption")] - None, ) .unwrap(); @@ -2317,8 +2311,6 @@ mod tests { r.rows_written as usize, None, Arc::new(props), - #[cfg(feature = "encryption")] - None, ) .unwrap(), ); @@ -3543,7 +3535,7 @@ mod tests { let _file_metadata = writer.close().unwrap(); let decryption_properties = FileDecryptionProperties::builder(footer_key.to_vec()) - .with_column_key(b"a".to_vec(), column_key.key().clone()) + .with_column_key("a", column_key.key().clone()) .build() .unwrap(); let options = ArrowReaderOptions::default() @@ -3955,8 +3947,6 @@ mod tests { result.rows_written as usize, None, Arc::new(props), - #[cfg(feature = "encryption")] - None, ) .unwrap(), ); diff --git c/parquet/src/encryption/ciphers.rs i/parquet/src/encryption/ciphers.rs index 88ec2a743..9c6ce23f8 100644 --- c/parquet/src/encryption/ciphers.rs +++ i/parquet/src/encryption/ciphers.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::errors::ParquetError; use crate::errors::ParquetError::General; use crate::errors::Result; use ring::aead::{Aad, LessSafeKey, NonceSequence, UnboundKey, AES_128_GCM}; @@ -167,7 +168,7 @@ mod tests { fn test_round_trip() { let key = [0u8; 16]; let mut encryptor = RingGcmBlockEncryptor::new(&key).unwrap(); - let decryptor = RingGcmBlockDecryptor::new(&key); + let decryptor = RingGcmBlockDecryptor::new(&key).unwrap(); let plaintext = b"hello, world!"; let aad = b"some aad"; diff --git c/parquet/src/encryption/decryption.rs i/parquet/src/encryption/decryption.rs deleted file mode 100644 index 2c789d020..000000000 --- c/parquet/src/encryption/decryption.rs +++ /dev/null @@ -1,256 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::encryption::ciphers::{BlockDecryptor, RingGcmBlockDecryptor}; -use crate::encryption::modules::{create_module_aad, ModuleType}; -use crate::errors::Result; -use std::collections::HashMap; -use std::io::Read; -use std::sync::Arc; - -pub fn read_and_decrypt<T: Read>( - decryptor: &Arc<dyn BlockDecryptor>, - input: &mut T, - aad: &[u8], -) -> Result<Vec<u8>> { - let mut len_bytes = [0; 4]; - input.read_exact(&mut len_bytes)?; - let ciphertext_len = u32::from_le_bytes(len_bytes) as usize; - let mut ciphertext = vec![0; 4 + ciphertext_len]; - ciphertext[0..4].copy_from_slice(&len_bytes); - input.read_exact(&mut ciphertext[4..])?; - - decryptor.decrypt(&ciphertext, aad.as_ref()) -} - -#[derive(Debug, Clone)] -pub struct CryptoContext { - pub(crate) row_group_ordinal: usize, - pub(crate) column_ordinal: usize, - pub(crate) page_ordinal: Option<usize>, - pub(crate) dictionary_page: bool, - // We have separate data and metadata decryptors because - // in GCM CTR mode, the metadata and data pages use - // different algorithms. - data_decryptor: Arc<dyn BlockDecryptor>, - metadata_decryptor: Arc<dyn BlockDecryptor>, - file_aad: Vec<u8>, -} - -impl CryptoContext { - pub fn new( - row_group_ordinal: usize, - column_ordinal: usize, - data_decryptor: Arc<dyn BlockDecryptor>, - metadata_decryptor: Arc<dyn BlockDecryptor>, - file_aad: Vec<u8>, - ) -> Self { - Self { - row_group_ordinal, - column_ordinal, - page_ordinal: None, - dictionary_page: false, - data_decryptor, - metadata_decryptor, - file_aad, - } - } - - pub fn with_page_ordinal(&self, page_ordinal: usize) -> Self { - Self { - row_group_ordinal: self.row_group_ordinal, - column_ordinal: self.column_ordinal, - page_ordinal: Some(page_ordinal), - dictionary_page: false, - data_decryptor: self.data_decryptor.clone(), - metadata_decryptor: self.metadata_decryptor.clone(), - file_aad: self.file_aad.clone(), - } - } - - pub(crate) fn create_page_header_aad(&self) -> Result<Vec<u8>> { - let module_type = if self.dictionary_page { - ModuleType::DictionaryPageHeader - } else { - ModuleType::DataPageHeader - }; - - create_module_aad( - self.file_aad(), - module_type, - self.row_group_ordinal, - self.column_ordinal, - self.page_ordinal, - ) - } - - pub(crate) fn create_page_aad(&self) -> Result<Vec<u8>> { - let module_type = if self.dictionary_page { - ModuleType::DictionaryPage - } else { - ModuleType::DataPage - }; - - create_module_aad( - self.file_aad(), - module_type, - self.row_group_ordinal, - self.column_ordinal, - self.page_ordinal, - ) - } - - pub fn for_dictionary_page(&self) -> Self { - Self { - row_group_ordinal: self.row_group_ordinal, - column_ordinal: self.column_ordinal, - page_ordinal: self.page_ordinal, - dictionary_page: true, - data_decryptor: self.data_decryptor.clone(), - metadata_decryptor: self.metadata_decryptor.clone(), - file_aad: self.file_aad.clone(), - } - } - - pub fn data_decryptor(&self) -> &Arc<dyn BlockDecryptor> { - &self.data_decryptor - } - - pub fn metadata_decryptor(&self) -> &Arc<dyn BlockDecryptor> { - &self.metadata_decryptor - } - - pub fn file_aad(&self) -> &Vec<u8> { - &self.file_aad - } -} - -/// FileDecryptionProperties hold keys and AAD data required to decrypt a Parquet file. -#[derive(Debug, Clone, PartialEq)] -pub struct FileDecryptionProperties { - footer_key: Vec<u8>, - column_keys: Option<HashMap<Vec<u8>, Vec<u8>>>, - aad_prefix: Option<Vec<u8>>, -} - -impl FileDecryptionProperties { - /// Returns a new FileDecryptionProperties builder - pub fn builder(footer_key: Vec<u8>) -> DecryptionPropertiesBuilder { - DecryptionPropertiesBuilder::new(footer_key) - } -} - -pub struct DecryptionPropertiesBuilder { - footer_key: Vec<u8>, - column_keys: Option<HashMap<Vec<u8>, Vec<u8>>>, - aad_prefix: Option<Vec<u8>>, -} - -impl DecryptionPropertiesBuilder { - pub fn new(footer_key: Vec<u8>) -> DecryptionPropertiesBuilder { - Self { - footer_key, - column_keys: None, - aad_prefix: None, - } - } - - pub fn build(self) -> Result<FileDecryptionProperties> { - Ok(FileDecryptionProperties { - footer_key: self.footer_key, - column_keys: self.column_keys, - aad_prefix: self.aad_prefix, - }) - } - - pub fn with_aad_prefix(mut self, value: Vec<u8>) -> Self { - self.aad_prefix = Some(value); - self - } - - pub fn with_column_key(mut self, column_name: Vec<u8>, decryption_key: Vec<u8>) -> Self { - let mut column_keys = self.column_keys.unwrap_or_default(); - column_keys.insert(column_name, decryption_key); - self.column_keys = Some(column_keys); - self - } -} - -#[derive(Clone, Debug)] -pub struct FileDecryptor { - decryption_properties: FileDecryptionProperties, - footer_decryptor: Option<Arc<dyn BlockDecryptor>>, - file_aad: Vec<u8>, -} - -impl PartialEq for FileDecryptor { - fn eq(&self, other: &Self) -> bool { - self.decryption_properties == other.decryption_properties - } -} - -impl FileDecryptor { - pub(crate) fn new( - decryption_properties: &FileDecryptionProperties, - aad_file_unique: Vec<u8>, - aad_prefix: Vec<u8>, - ) -> Self { - let file_aad = [aad_prefix.as_slice(), aad_file_unique.as_slice()].concat(); - let footer_decryptor = RingGcmBlockDecryptor::new(&decryption_properties.footer_key); - - Self { - // todo decr: if no key available yet (not set in properties, will be retrieved from metadata) - footer_decryptor: Some(Arc::new(footer_decryptor)), - decryption_properties: decryption_properties.clone(), - file_aad, - } - } - - pub(crate) fn get_footer_decryptor(&self) -> Arc<dyn BlockDecryptor> { - self.footer_decryptor.clone().unwrap() - } - - pub(crate) fn get_column_data_decryptor(&self, column_name: &[u8]) -> Arc<dyn BlockDecryptor> { - match self.decryption_properties.column_keys.as_ref() { - None => self.get_footer_decryptor(), - Some(column_keys) => match column_keys.get(column_name) { - None => self.get_footer_decryptor(), - Some(column_key) => Arc::new(RingGcmBlockDecryptor::new(column_key)), - }, - } - } - - pub(crate) fn get_column_metadata_decryptor( - &self, - column_name: &[u8], - ) -> Arc<dyn BlockDecryptor> { - // Once GCM CTR mode is implemented, data and metadata decryptors may be different - self.get_column_data_decryptor(column_name) - } - - pub(crate) fn file_aad(&self) -> &Vec<u8> { - &self.file_aad - } - - pub(crate) fn is_column_encrypted(&self, column_name: &[u8]) -> bool { - // Column is encrypted if either uniform encryption is used or an encryption key is set for the column - match self.decryption_properties.column_keys.as_ref() { - None => true, - Some(keys) => keys.contains_key(column_name), - } - } -} diff --git c/parquet/src/file/footer.rs i/parquet/src/file/footer.rs index 5be084259..85ef30cd0 100644 --- c/parquet/src/file/footer.rs +++ i/parquet/src/file/footer.rs @@ -58,12 +58,7 @@ pub fn parse_metadata<R: ChunkReader>(chunk_reader: &R) -> Result<ParquetMetaDat /// [Parquet Spec]: https://github.com/apache/parquet-format#metadata #[deprecated(since = "53.1.0", note = "Use ParquetMetaDataReader::decode_metadata")] pub fn decode_metadata(buf: &[u8]) -> Result<ParquetMetaData> { - ParquetMetaDataReader::decode_metadata( - buf, - false, - #[cfg(feature = "encryption")] - None, - ) + ParquetMetaDataReader::decode_metadata(buf) } /// Decodes the Parquet footer returning the metadata length in bytes diff --git c/parquet/src/file/metadata/mod.rs i/parquet/src/file/metadata/mod.rs index 6a5eb286d..6dc559c8b 100644 --- c/parquet/src/file/metadata/mod.rs +++ i/parquet/src/file/metadata/mod.rs @@ -103,14 +103,17 @@ use crate::encryption::{ }; use crate::errors::{ParquetError, Result}; #[cfg(feature = "encryption")] -use crate::file::column_crypto_metadata::{self, ColumnCryptoMetaData}; +use crate::file::column_crypto_metadata::ColumnCryptoMetaData; +#[cfg(feature = "encryption")] +use crate::format::ColumnCryptoMetaData as TColumnCryptoMetaData; + +#[cfg(feature = "encryption")] +use crate::file::column_crypto_metadata; pub(crate) use crate::file::metadata::memory::HeapSize; use crate::file::page_encoding_stats::{self, PageEncodingStats}; use crate::file::page_index::index::Index; use crate::file::page_index::offset_index::OffsetIndexMetaData; use crate::file::statistics::{self, Statistics}; -#[cfg(feature = "encryption")] -use crate::format::ColumnCryptoMetaData as TColumnCryptoMetaData; use crate::format::{ BoundaryOrder, ColumnChunk, ColumnIndex, ColumnMetaData, OffsetIndex, PageLocation, RowGroup, SizeStatistics, SortingColumn, @@ -659,11 +662,11 @@ impl RowGroupMetaData { d.path().string() )); } - Some(ColumnCryptoMetaData::ENCRYPTIONWITHCOLUMNKEY(crypto_metadata)) => { + Some(TColumnCryptoMetaData::ENCRYPTIONWITHCOLUMNKEY(crypto_metadata)) => { let column_name = crypto_metadata.path_in_schema.join("."); decryptor.get_column_metadata_decryptor(column_name.as_str())? } - Some(ColumnCryptoMetaData::ENCRYPTIONWITHFOOTERKEY(_)) => { + Some(TColumnCryptoMetaData::ENCRYPTIONWITHFOOTERKEY(_)) => { decryptor.get_footer_decryptor()? } }; diff --git c/parquet/src/file/writer.rs i/parquet/src/file/writer.rs index 36c75d3f1..94089824b 100644 --- c/parquet/src/file/writer.rs +++ i/parquet/src/file/writer.rs @@ -1467,8 +1467,6 @@ mod tests { total_num_values as usize, None, Arc::new(props), - #[cfg(feature = "encryption")] - None, ) .unwrap(); diff --git c/parquet/tests/arrow_reader/encryption_util.rs i/parquet/tests/arrow_reader/encryption_util.rs index 604cd3034..6dea55712 100644 --- c/parquet/tests/arrow_reader/encryption_util.rs +++ i/parquet/tests/arrow_reader/encryption_util.rs @@ -18,15 +18,20 @@ use crate::arrow::arrow_reader::{ ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, }; +use crate::arrow::ParquetRecordBatchStreamBuilder; use arrow_array::cast::AsArray; use arrow_array::{types, RecordBatch}; use std::fs::File; use parquet::file::metadata::ParquetMetaData; use crate::arrow::ArrowWriter; -use crate::encryption::encrypt::FileEncryptionProperties; use crate::encryption::decrypt::FileDecryptionProperties; +use crate::encryption::encrypt::FileEncryptionProperties; +use crate::errors::ParquetError; +use crate::file::metadata::FileMetaData; use crate::file::properties::WriterProperties; +use futures::TryStreamExt; +use std::fs::File; /// Tests reading an encrypted file from the parquet-testing repository pub(crate) fn verify_encryption_test_file_read( diff --git c/parquet/tests/arrow_writer_layout.rs i/parquet/tests/arrow_writer_layout.rs index 9297b8d13..9a66d13f8 100644 --- c/parquet/tests/arrow_writer_layout.rs +++ i/parquet/tests/arrow_writer_layout.rs @@ -141,8 +141,6 @@ fn assert_layout(file_reader: &Bytes, meta: &ParquetMetaData, layout: &Layout) { row_group.num_rows() as usize, None, Arc::new(properties), - #[cfg(feature = "encryption")] - None, ) .unwrap();

Co-authored-by: Adam Reeve <[email protected]>

parquet/src/encryption/encrypt.rs

adamreeve · 2025-03-18T01:01:02Z

parquet/src/file/writer.rs

+        #[cfg(feature = "encryption")]
+        if let Some(ref file_encryption_properties) = properties.file_encryption_properties {
+            if !file_encryption_properties.encrypt_footer() {
+                return Err(general_err!("Footer encryption is not supported yet"));


Suggested change

return Err(general_err!("Footer encryption is not supported yet"));

return Err(general_err!("Writing encrypted files with plaintext footers is not supported yet"));

parquet/src/file/writer.rs

github-actions bot added the parquet Changes to the parquet crate label Feb 10, 2025

rok force-pushed the encryption-basics-fork branch 5 times, most recently from 8e6a4be to d1ccf75 Compare February 17, 2025 18:52

rok force-pushed the encryption-basics-fork branch 3 times, most recently from f6d1155 to 05d4d60 Compare February 21, 2025 12:18

adamreeve force-pushed the encryption-basics-fork branch 3 times, most recently from dad9642 to 5d52ef7 Compare March 4, 2025 07:08

rok force-pushed the encryption-basics-fork branch 2 times, most recently from 36d7c70 to bc75007 Compare March 4, 2025 19:06

adamreeve force-pushed the encryption-basics-fork branch 3 times, most recently from 24e70d9 to cc58a3a Compare March 6, 2025 00:11

rok force-pushed the encryption-basics-fork branch 2 times, most recently from 65fca4b to 44559b0 Compare March 6, 2025 11:26

adamreeve mentioned this pull request Mar 9, 2025

Support Parquet key management tools #7256

Open

rok force-pushed the encryption-basics-fork branch 2 times, most recently from c5a692c to 87224ea Compare March 10, 2025 20:05

rok mentioned this pull request Mar 10, 2025

Parquet Modular Encryption support #3511

Closed

rok force-pushed the encryption-basics-fork branch 4 times, most recently from cab7dd4 to de5feef Compare March 11, 2025 12:50

This was referenced Mar 12, 2025

Add Parquet Modular decryption (read) support + encryption flag #6637

Merged

[Epic] Parquet Modular Encryption Support #7278

Open

alamb changed the title ~~Parquet Modular encryption support~~ Add Parquet Modular encryption support (write) Mar 12, 2025

rok and others added 6 commits March 16, 2025 02:19

Check if columns to encrypt are in schema

854b257

Apply suggestions from code review

ea2db66

Co-authored-by: Adam Reeve <[email protected]>

Move tests to tests/. Post rebase fixes.

2b2a3ef

Review feedback

22135ba

rok force-pushed the encryption-basics-fork branch 3 times, most recently from 6ad8dd1 to 294689a Compare March 16, 2025 01:40

Review feedback

32673c1

rok force-pushed the encryption-basics-fork branch from 294689a to 32673c1 Compare March 16, 2025 02:10

Minor changes

f479df1

rok force-pushed the encryption-basics-fork branch from 04ce83e to 7fe2afc Compare March 16, 2025 14:44

Raise if writing plaintext footer

8ac232b

rok force-pushed the encryption-basics-fork branch from 7fe2afc to 8ac232b Compare March 16, 2025 14:49

Docs for crypto methods

75b98cd

rok force-pushed the encryption-basics-fork branch from 4aa657b to 75b98cd Compare March 16, 2025 16:07

rok requested a review from adamreeve March 16, 2025 16:18

alamb mentioned this pull request Mar 17, 2025

Weekly Plan (Andrew Lamb) March 17, 2025 apache/datafusion#15274

Open

10 tasks

corwinjoy reviewed Mar 17, 2025

View reviewed changes

parquet/src/encryption/encrypt.rs Show resolved Hide resolved

rok force-pushed the encryption-basics-fork branch 5 times, most recently from 327af72 to baa6e54 Compare March 17, 2025 22:56

More practical key API

0460424

rok force-pushed the encryption-basics-fork branch from baa6e54 to 0460424 Compare March 17, 2025 23:00

adamreeve reviewed Mar 18, 2025

View reviewed changes

parquet/src/file/writer.rs Outdated Show resolved Hide resolved

Refactor PageEncryptor use

4032b4f

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add Parquet Modular encryption support (write) #7111

Add Parquet Modular encryption support (write) #7111

rok commented Feb 10, 2025

adamreeve Mar 18, 2025

	return Err(general_err!("Footer encryption is not supported yet"));
	return Err(general_err!("Writing encrypted files with plaintext footers is not supported yet"));

Add Parquet Modular encryption support (write) #7111

Are you sure you want to change the base?

Add Parquet Modular encryption support (write) #7111

Conversation

rok commented Feb 10, 2025

Which issue does this PR close?

Rationale for this change

What changes are included in this PR?

Are there any user-facing changes?

adamreeve Mar 18, 2025

Choose a reason for hiding this comment