Skip to content

Commit 4f195d0

Browse files
authored
feat(encryption): add AES GCM cipher and AES GCM Stream (AGS1) files (#1030)
1 parent 08d2f86 commit 4f195d0

12 files changed

Lines changed: 1162 additions & 19 deletions

‎ci/scripts/generate_public_headers.py‎

Lines changed: 39 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,21 @@
1919

2020
"""Regenerate src/iceberg/cpp20_compatibility_internal.h.
2121
22-
The generated header includes every public header, i.e. every header that
23-
iceberg_install_all_headers() installs: *.h files in a directory whose
24-
CMakeLists.txt calls it, except those with "internal" in their name. This script
25-
applies the same rule, so the list never has to be maintained by hand. The
26-
generate-public-headers pre-commit hook runs it and fails when the checked-in
27-
header is out of date.
28-
29-
Headers of optional components are wrapped in a macro that
30-
cpp20_compatibility_test defines only when that component is built.
22+
The generated header covers all possible public headers installed by
23+
iceberg_install_all_headers(): *.h and *.hpp files matching its PATTERN arguments,
24+
except those with "internal" in their name. The generate-public-headers pre-commit
25+
hook runs this script and fails when the checked-in header is out of date.
26+
27+
Headers of optional components are grouped by directory or an explicit
28+
HEADER_GROUP in the installation call. cpp20_compatibility_test defines each
29+
group's macro only when the corresponding component is built.
3130
"""
3231

3332
from __future__ import annotations
3433

34+
import fnmatch
3535
import re
36+
import shlex
3637
import sys
3738
from pathlib import Path
3839

@@ -51,7 +52,7 @@
5152
("iceberg/parquet", "ICEBERG_PUBLIC_HEADERS_BUNDLE"),
5253
]
5354

54-
INSTALL_CALL = re.compile(r"iceberg_install_all_headers\(\s*([^\s)]+)")
55+
INSTALL_CALL = re.compile(r"iceberg_install_all_headers\((.*?)\)", re.DOTALL)
5556

5657
LICENSE = """\
5758
/*
@@ -75,7 +76,9 @@
7576
"""
7677

7778

78-
def component_of(install_path: str) -> str | None:
79+
def component_of(install_path: str, header_group: str | None = None) -> str | None:
80+
if header_group:
81+
return header_group
7982
for prefix, macro in OPTIONAL_COMPONENTS:
8083
if install_path == prefix or install_path.startswith(prefix + "/"):
8184
return macro
@@ -85,13 +88,29 @@ def component_of(install_path: str) -> str | None:
8588
def collect_public_headers() -> dict[str | None, list[str]]:
8689
headers: dict[str | None, list[str]] = {}
8790
for cmake_file in sorted(SRC.rglob("CMakeLists.txt")):
88-
for install_path in INSTALL_CALL.findall(cmake_file.read_text()):
91+
for call in INSTALL_CALL.findall(cmake_file.read_text()):
92+
tokens = shlex.split(call, comments=True)
93+
if not tokens:
94+
continue
95+
install_path = tokens[0]
96+
try:
97+
pattern_start = tokens.index("PATTERN") + 1
98+
except ValueError:
99+
pattern_start = len(tokens)
100+
try:
101+
group_start = tokens.index("HEADER_GROUP")
102+
header_group = tokens[group_start + 1]
103+
except ValueError:
104+
group_start = len(tokens)
105+
header_group = None
106+
patterns = tokens[pattern_start:group_start] or ["*.h", "*.hpp"]
107+
group = component_of(install_path, header_group)
89108
for header in cmake_file.parent.iterdir():
90-
if header.suffix != ".h" or "internal" in header.name:
109+
if not any(fnmatch.fnmatch(header.name, pattern) for pattern in patterns):
91110
continue
92-
headers.setdefault(component_of(install_path), []).append(
93-
f"{install_path}/{header.name}"
94-
)
111+
if header.suffix not in {".h", ".hpp"} or "internal" in header.name:
112+
continue
113+
headers.setdefault(group, []).append(f"{install_path}/{header.name}")
95114
return headers
96115

97116

@@ -107,7 +126,10 @@ def render(headers: dict[str | None, list[str]]) -> str:
107126
"",
108127
]
109128
lines += [f'#include "{h}"' for h in sorted(headers.get(None, []))]
110-
for macro in sorted({m for _, m in OPTIONAL_COMPONENTS}):
129+
optional_macros = {m for _, m in OPTIONAL_COMPONENTS} | {
130+
group for group in headers if group is not None
131+
}
132+
for macro in sorted(optional_macros):
111133
if macro not in headers:
112134
continue
113135
lines += ["", f"#ifdef {macro}"]

‎cmake_modules/IcebergBuildUtils.cmake‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,8 @@ endfunction()
275275

276276
function(iceberg_install_all_headers PATH)
277277
set(options)
278-
set(one_value_args)
278+
# HEADER_GROUP labels optional headers for ci/scripts/generate_public_headers.py.
279+
set(one_value_args HEADER_GROUP)
279280
set(multi_value_args PATTERN)
280281
cmake_parse_arguments(ARG
281282
"${options}"

‎src/iceberg/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,10 @@ if(ICEBERG_BUILD_BUNDLE)
326326
parquet/parquet_writer.cc
327327
parquet/row_number_reader.cc)
328328

329+
if(ICEBERG_ENCRYPTION)
330+
list(APPEND ICEBERG_BUNDLE_SOURCES encryption/aes_gcm.cc encryption/aes_gcm_stream.cc)
331+
endif()
332+
329333
# Libraries to link with exported libiceberg_bundle.{so,a}.
330334
set(ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS)
331335
set(ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS)

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,10 @@
216216
# include "iceberg/parquet/parquet_writer.h"
217217
#endif // ICEBERG_PUBLIC_HEADERS_BUNDLE
218218

219+
#ifdef ICEBERG_PUBLIC_HEADERS_ENCRYPTION
220+
# include "iceberg/encryption/aes_gcm_stream.h"
221+
#endif // ICEBERG_PUBLIC_HEADERS_ENCRYPTION
222+
219223
#ifdef ICEBERG_PUBLIC_HEADERS_HIVE
220224
# include "iceberg/catalog/hive/hive_catalog.h"
221225
# include "iceberg/catalog/hive/hive_catalog_properties.h"

‎src/iceberg/encryption/CMakeLists.txt‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,12 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
iceberg_install_all_headers(iceberg/encryption)
18+
iceberg_install_all_headers(iceberg/encryption PATTERN encrypted_key.h
19+
standard_key_metadata.h)
20+
if(ICEBERG_BUILD_BUNDLE AND ICEBERG_ENCRYPTION)
21+
iceberg_install_all_headers(iceberg/encryption
22+
PATTERN
23+
aes_gcm_stream.h
24+
HEADER_GROUP
25+
ICEBERG_PUBLIC_HEADERS_ENCRYPTION)
26+
endif()

‎src/iceberg/encryption/aes_gcm.cc‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include <limits>
21+
22+
#include <openssl/evp.h>
23+
#include <openssl/rand.h>
24+
25+
#include "iceberg/encryption/aes_gcm_internal.h"
26+
#include "iceberg/util/macros.h"
27+
28+
namespace iceberg::encryption {
29+
30+
namespace {
31+
32+
const EVP_CIPHER* CipherForKey(size_t key_length) {
33+
switch (key_length) {
34+
case 16:
35+
return EVP_aes_128_gcm();
36+
case 24:
37+
return EVP_aes_192_gcm();
38+
case 32:
39+
return EVP_aes_256_gcm();
40+
default:
41+
return nullptr;
42+
}
43+
}
44+
45+
EVP_CIPHER_CTX* Ctx(void* ctx) { return static_cast<EVP_CIPHER_CTX*>(ctx); }
46+
47+
} // namespace
48+
49+
Status SecureRandomBytes(std::span<uint8_t> out) {
50+
if (out.size() > static_cast<size_t>(std::numeric_limits<int>::max()) ||
51+
RAND_bytes(out.data(), static_cast<int>(out.size())) != 1) {
52+
return IOError("Failed to generate secure random bytes");
53+
}
54+
return {};
55+
}
56+
57+
Result<std::unique_ptr<AesGcmCipher>> AesGcmCipher::Make(std::span<const uint8_t> key) {
58+
if (CipherForKey(key.size()) == nullptr) {
59+
return InvalidArgument("Invalid key length: {} (must be 16, 24, or 32 bytes)",
60+
key.size());
61+
}
62+
EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
63+
if (ctx == nullptr) {
64+
return IOError("Failed to create cipher context");
65+
}
66+
return std::unique_ptr<AesGcmCipher>(
67+
new AesGcmCipher(std::vector<uint8_t>(key.begin(), key.end()), ctx));
68+
}
69+
70+
AesGcmCipher::AesGcmCipher(std::vector<uint8_t> key, void* ctx)
71+
: key_(std::move(key)), ctx_(ctx) {}
72+
73+
AesGcmCipher::~AesGcmCipher() { EVP_CIPHER_CTX_free(Ctx(ctx_)); }
74+
75+
Status AesGcmCipher::Encrypt(std::span<const uint8_t> plaintext,
76+
std::span<const uint8_t> aad, std::span<uint8_t> out) {
77+
if (out.size() != plaintext.size() + kNonceLength + kGcmTagLength) {
78+
return InvalidArgument("Invalid ciphertext buffer size: {}", out.size());
79+
}
80+
if (plaintext.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
81+
return InvalidArgument("Plaintext too large: {}", plaintext.size());
82+
}
83+
84+
auto nonce = out.first(kNonceLength);
85+
auto ciphertext = out.subspan(kNonceLength, plaintext.size());
86+
auto tag = out.last(kGcmTagLength);
87+
ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(nonce));
88+
89+
auto* ctx = Ctx(ctx_);
90+
int len = 0;
91+
if (EVP_EncryptInit_ex(ctx, CipherForKey(key_.size()), /*impl=*/nullptr,
92+
/*key=*/nullptr, /*iv=*/nullptr) != 1 ||
93+
EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, /*ptr=*/nullptr) !=
94+
1 ||
95+
EVP_EncryptInit_ex(ctx, /*cipher=*/nullptr, /*impl=*/nullptr, key_.data(),
96+
nonce.data()) != 1) {
97+
return IOError("Failed to initialize AES GCM encryption");
98+
}
99+
if (!aad.empty() && EVP_EncryptUpdate(ctx, /*out=*/nullptr, &len, aad.data(),
100+
static_cast<int>(aad.size())) != 1) {
101+
return IOError("Failed to set AES GCM AAD");
102+
}
103+
if (!plaintext.empty() &&
104+
EVP_EncryptUpdate(ctx, ciphertext.data(), &len, plaintext.data(),
105+
static_cast<int>(plaintext.size())) != 1) {
106+
return IOError("Failed to encrypt");
107+
}
108+
if (EVP_EncryptFinal_ex(ctx, ciphertext.data() + plaintext.size(), &len) != 1 ||
109+
EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, kGcmTagLength, tag.data()) != 1) {
110+
return IOError("Failed to finalize AES GCM encryption");
111+
}
112+
return {};
113+
}
114+
115+
Status AesGcmCipher::Decrypt(std::span<const uint8_t> ciphertext,
116+
std::span<const uint8_t> aad, std::span<uint8_t> out) {
117+
if (ciphertext.size() < static_cast<size_t>(kNonceLength + kGcmTagLength)) {
118+
return Invalid(
119+
"Cannot decrypt cipher text of length {}: shorter than nonce and tag, text may "
120+
"not be encrypted with AES GCM",
121+
ciphertext.size());
122+
}
123+
size_t plaintext_length = ciphertext.size() - kNonceLength - kGcmTagLength;
124+
if (out.size() != plaintext_length) {
125+
return InvalidArgument("Invalid plaintext buffer size: {}", out.size());
126+
}
127+
128+
auto nonce = ciphertext.first(kNonceLength);
129+
auto data = ciphertext.subspan(kNonceLength, plaintext_length);
130+
auto tag = ciphertext.last(kGcmTagLength);
131+
132+
auto* ctx = Ctx(ctx_);
133+
int len = 0;
134+
if (EVP_DecryptInit_ex(ctx, CipherForKey(key_.size()), /*impl=*/nullptr,
135+
/*key=*/nullptr, /*iv=*/nullptr) != 1 ||
136+
EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, /*ptr=*/nullptr) !=
137+
1 ||
138+
EVP_DecryptInit_ex(ctx, /*cipher=*/nullptr, /*impl=*/nullptr, key_.data(),
139+
nonce.data()) != 1) {
140+
return IOError("Failed to initialize AES GCM decryption");
141+
}
142+
if (!aad.empty() && EVP_DecryptUpdate(ctx, /*out=*/nullptr, &len, aad.data(),
143+
static_cast<int>(aad.size())) != 1) {
144+
return IOError("Failed to set AES GCM AAD");
145+
}
146+
if (!data.empty() && EVP_DecryptUpdate(ctx, out.data(), &len, data.data(),
147+
static_cast<int>(data.size())) != 1) {
148+
return IOError("Failed to decrypt");
149+
}
150+
if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, kGcmTagLength,
151+
const_cast<uint8_t*>(tag.data())) != 1 ||
152+
EVP_DecryptFinal_ex(ctx, out.data() + plaintext_length, &len) != 1) {
153+
return Invalid(
154+
"GCM tag check failed. Possible reasons: wrong decryption key; or "
155+
"corrupt/tampered data. AES GCM doesn't differentiate between these two.");
156+
}
157+
return {};
158+
}
159+
160+
Result<std::vector<uint8_t>> AesGcmCipher::Encrypt(std::span<const uint8_t> plaintext,
161+
std::span<const uint8_t> aad) {
162+
std::vector<uint8_t> out(plaintext.size() + kNonceLength + kGcmTagLength);
163+
ICEBERG_RETURN_UNEXPECTED(Encrypt(plaintext, aad, out));
164+
return out;
165+
}
166+
167+
Result<std::vector<uint8_t>> AesGcmCipher::Decrypt(std::span<const uint8_t> ciphertext,
168+
std::span<const uint8_t> aad) {
169+
if (ciphertext.size() < static_cast<size_t>(kNonceLength + kGcmTagLength)) {
170+
return Invalid("Cannot decrypt cipher text of length {}", ciphertext.size());
171+
}
172+
std::vector<uint8_t> out(ciphertext.size() - kNonceLength - kGcmTagLength);
173+
ICEBERG_RETURN_UNEXPECTED(Decrypt(ciphertext, aad, out));
174+
return out;
175+
}
176+
177+
} // namespace iceberg::encryption
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
/// \file iceberg/encryption/aes_gcm_internal.h
23+
/// AES GCM cipher compatible with Java's org.apache.iceberg.encryption.Ciphers.
24+
25+
#include <cstdint>
26+
#include <memory>
27+
#include <span>
28+
#include <vector>
29+
30+
#include "iceberg/result.h"
31+
32+
namespace iceberg::encryption {
33+
34+
inline constexpr int32_t kNonceLength = 12;
35+
inline constexpr int32_t kGcmTagLength = 16;
36+
37+
/// \brief Fill `out` with cryptographically secure random bytes.
38+
Status SecureRandomBytes(std::span<uint8_t> out);
39+
40+
/// \brief AES GCM with a random 12-byte nonce.
41+
///
42+
/// Ciphertext layout: nonce (12) | encrypted data | tag (16).
43+
class AesGcmCipher {
44+
public:
45+
/// \brief Create a cipher for a 16, 24 or 32 byte key.
46+
static Result<std::unique_ptr<AesGcmCipher>> Make(std::span<const uint8_t> key);
47+
48+
~AesGcmCipher();
49+
50+
/// \brief Encrypt `plaintext` into `out`, which must hold plaintext.size() + 28 bytes.
51+
Status Encrypt(std::span<const uint8_t> plaintext, std::span<const uint8_t> aad,
52+
std::span<uint8_t> out);
53+
54+
/// \brief Decrypt and verify `ciphertext` into `out`, which must hold
55+
/// ciphertext.size() - 28 bytes.
56+
Status Decrypt(std::span<const uint8_t> ciphertext, std::span<const uint8_t> aad,
57+
std::span<uint8_t> out);
58+
59+
Result<std::vector<uint8_t>> Encrypt(std::span<const uint8_t> plaintext,
60+
std::span<const uint8_t> aad);
61+
Result<std::vector<uint8_t>> Decrypt(std::span<const uint8_t> ciphertext,
62+
std::span<const uint8_t> aad);
63+
64+
private:
65+
AesGcmCipher(std::vector<uint8_t> key, void* ctx);
66+
67+
std::vector<uint8_t> key_;
68+
void* ctx_; // EVP_CIPHER_CTX
69+
};
70+
71+
} // namespace iceberg::encryption

0 commit comments

Comments
 (0)