Skip to content

Commit 00d1c5b

Browse files
urykhyurykhy
committed
test encrypted zone support
1 parent 4d0d7ec commit 00d1c5b

File tree

6 files changed

+225
-6
lines changed

6 files changed

+225
-6
lines changed

‎.github/scripts/fixtures.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,13 @@ set -e
33
hadoop fs -mkdir -p "/_test"
44
hadoop fs -chmod 777 "/_test"
55

6+
if [ "$TRANSPARENT_ENCRYPTION" = "true" ]; then
7+
echo "Prepare encrypted zone"
8+
hadoop fs -mkdir /_test/kms
9+
hadoop fs -chmod 777 "/_test/kms"
10+
hadoop key create key1
11+
hdfs crypto -createZone -keyName key1 -path /_test/kms
12+
fi
13+
614
hadoop fs -put ./testdata/foo.txt "/_test/foo.txt"
715
hadoop fs -Ddfs.block.size=1048576 -put ./testdata/mobydick.txt "/_test/mobydick.txt"

‎.github/scripts/install-hdfs.sh‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set -e
44

5-
KERBEROS=${KERBEROS-"false"}
5+
KERBEROS="${KERBEROS-false}"
66
AES=${AES-"false"}
77
if [ "$DATA_TRANSFER_PROTECTION" = "privacy" ]; then
88
KERBEROS="true"
@@ -15,11 +15,18 @@ else
1515
ENCRYPT_DATA_TRANSFER="false"
1616
fi
1717

18+
CONF_KMS_PROVIDER=""
19+
TRANSPARENT_ENCRYPTION=false
20+
if [ "$HADOOP_VERSION" != "2.10.1" ]; then
21+
TRANSPARENT_ENCRYPTION=true
22+
CONF_KMS_PROVIDER="kms://http@localhost:9600/kms"
23+
fi
24+
1825
CONF_AUTHENTICATION="simple"
1926
KERBEROS_REALM="EXAMPLE.COM"
2027
KERBEROS_PRINCIPLE="administrator"
2128
KERBEROS_PASSWORD="password1234"
22-
if [ $KERBEROS = "true" ]; then
29+
if [ "$KERBEROS" = "true" ]; then
2330
CONF_AUTHENTICATION="kerberos"
2431

2532
HOSTNAME=$(hostname)
@@ -50,13 +57,16 @@ EOF
5057
sudo apt-get install -y krb5-user krb5-kdc krb5-admin-server
5158

5259
printf "$KERBEROS_PASSWORD\n$KERBEROS_PASSWORD" | sudo kdb5_util -r "$KERBEROS_REALM" create -s
53-
for p in nn dn $USER gohdfs1 gohdfs2; do
60+
for p in nn dn kms $USER gohdfs1 gohdfs2; do
5461
sudo kadmin.local -q "addprinc -randkey $p/$HOSTNAME@$KERBEROS_REALM"
5562
sudo kadmin.local -q "addprinc -randkey $p/localhost@$KERBEROS_REALM"
5663
sudo kadmin.local -q "xst -k /tmp/$p.keytab $p/$HOSTNAME@$KERBEROS_REALM"
5764
sudo kadmin.local -q "xst -k /tmp/$p.keytab $p/localhost@$KERBEROS_REALM"
5865
sudo chmod +rx /tmp/$p.keytab
5966
done
67+
# HTTP service for KMS
68+
sudo kadmin.local -q "addprinc -randkey HTTP/localhost@$KERBEROS_REALM"
69+
sudo kadmin.local -q "xst -k /tmp/kms.keytab HTTP/localhost@$KERBEROS_REALM"
6070

6171
echo "Restarting krb services..."
6272
sudo service krb5-kdc restart
@@ -116,6 +126,10 @@ sudo tee $HADOOP_ROOT/etc/hadoop/core-site.xml <<EOF
116126
<name>hadoop.rpc.protection</name>
117127
<value>$RPC_PROTECTION</value>
118128
</property>
129+
<property>
130+
<name>hadoop.security.key.provider.path</name>
131+
<value>$CONF_KMS_PROVIDER</value>
132+
</property>
119133
</configuration>
120134
EOF
121135

@@ -125,6 +139,10 @@ sudo tee $HADOOP_ROOT/etc/hadoop/hdfs-site.xml <<EOF
125139
<name>dfs.namenode.name.dir</name>
126140
<value>/tmp/hdfs/name</value>
127141
</property>
142+
<property>
143+
<name>dfs.namenode.fs-limits.min-block-size</name>
144+
<value>131072</value>
145+
</property>
128146
<property>
129147
<name>dfs.datanode.data.dir</name>
130148
<value>/tmp/hdfs/data</value>
@@ -172,6 +190,41 @@ $HADOOP_ROOT/bin/hdfs namenode -format
172190
sudo groupadd hadoop
173191
sudo usermod -a -G hadoop $USER
174192

193+
sudo tee $HADOOP_ROOT/etc/hadoop/kms-site.xml <<EOF
194+
<configuration>
195+
<property>
196+
<name>hadoop.kms.key.provider.uri</name>
197+
<value>jceks://file@/tmp/hdfs/kms.keystore</value>
198+
</property>
199+
<property>
200+
<name>hadoop.security.keystore.java-keystore-provider.password-file</name>
201+
<value>kms.keystore.password</value>
202+
</property>
203+
<property>
204+
<name>hadoop.kms.authentication.type</name>
205+
<value>$CONF_AUTHENTICATION</value>
206+
</property>
207+
<property>
208+
<name>hadoop.kms.authentication.kerberos.keytab</name>
209+
<value>/tmp/kms.keytab</value>
210+
</property>
211+
<property>
212+
<name>hadoop.kms.authentication.kerberos.principal</name>
213+
<value>HTTP/localhost@$KERBEROS_REALM</value>
214+
</property>
215+
</configuration>
216+
EOF
217+
218+
sudo tee $HADOOP_ROOT/etc/hadoop/kms.keystore.password <<EOF
219+
123456
220+
EOF
221+
222+
if [ "$TRANSPARENT_ENCRYPTION" = "true" ]; then
223+
echo "Starting KMS..."
224+
rm $HADOOP_ROOT/etc/hadoop/kms-log4j.properties
225+
$HADOOP_ROOT/bin/hadoop kms > /tmp/hdfs/kms.log 2>&1 &
226+
fi
227+
175228
echo "Starting namenode..."
176229
$HADOOP_ROOT/bin/hdfs namenode > /tmp/hdfs/namenode.log 2>&1 &
177230

@@ -184,4 +237,5 @@ echo "Waiting for cluster to exit safe mode..."
184237
$HADOOP_ROOT/bin/hdfs dfsadmin -safemode wait
185238

186239
echo "HADOOP_CONF_DIR=$(pwd)/$HADOOP_ROOT/etc/hadoop" >> $GITHUB_ENV
187-
echo "$(pwd)/$HADOOP_ROOT/bin" >> $GITHUB_PATH
240+
echo "TRANSPARENT_ENCRYPTION=$TRANSPARENT_ENCRYPTION" >> $GITHUB_ENV
241+
echo "$(pwd)/$HADOOP_ROOT/bin" >> $GITHUB_PATH

‎.github/workflows/tests.yml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ jobs:
3838
go-version: 1.17
3939

4040
# This step installs downloads hadoop and starts a local cluster with one
41-
# namenode and one datanode. It adds the hadoop binaries to GITHUB_PATH
42-
# and HADOOP_CONF_DIR to GITHUB_ENV.
41+
# namenode and one datanode. It adds the hadoop binaries to GITHUB_PATH,
42+
# TRANSPARENT_ENCRYPTION and HADOOP_CONF_DIR to GITHUB_ENV.
4343
- name: install-hdfs.sh
4444
run: ./.github/scripts/install-hdfs.sh
4545
env:
@@ -65,6 +65,16 @@ jobs:
6565
run: |
6666
make test
6767
68+
- name: cat kms.log
69+
if: always()
70+
run: |
71+
if [ -f /tmp/hdfs/kms.log ]
72+
then
73+
cat /tmp/hdfs/kms.log
74+
else
75+
echo "not exists"
76+
fi
77+
6878
- name: cat namenode.log
6979
if: always()
7080
run: cat /tmp/hdfs/namenode.log

‎cmd/hdfs/test/helper.bash‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/bin/bash
22

33
export HADOOP_FS=${HADOOP_FS-"hadoop fs"}
4+
export HADOOP_KEY=${HADOOP_KEY-"hadoop key"}
45
export ROOT_TEST_DIR="$BATS_TEST_DIRNAME/../../.."
56
export HDFS="$ROOT_TEST_DIR/hdfs"
67

‎cmd/hdfs/test/te.bats‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/usr/bin/env bats
2+
3+
load helper
4+
5+
@test "te: upload via native client, ensure we can download" {
6+
if [ "$TRANSPARENT_ENCRYPTION" = "true" ]; then
7+
run $HADOOP_FS -put $ROOT_TEST_DIR/testdata/foo.txt /_test/kms/foo1
8+
assert_success
9+
run $HDFS cat /_test/kms/foo1
10+
assert_output "bar"
11+
else
12+
skip
13+
fi
14+
}
15+
16+
@test "te: ensure native client can download once we uploaded to encrypted zone" {
17+
if [ "$TRANSPARENT_ENCRYPTION" = "true" ]; then
18+
run $HDFS put $ROOT_TEST_DIR/testdata/foo.txt /_test/kms/foo2
19+
assert_success
20+
run $HADOOP_FS -cat /_test/kms/foo2
21+
assert_output "bar"
22+
else
23+
skip
24+
fi
25+
}
26+
27+
@test "te: tail" {
28+
if [ "$TRANSPARENT_ENCRYPTION" = "true" ]; then
29+
run $HDFS put $ROOT_TEST_DIR/testdata/mobydick.txt /_test/kms/
30+
assert_success
31+
run bash -c "$HDFS tail /_test/kms/mobydick.txt > $BATS_TMPDIR/mobydick_test.txt"
32+
assert_success
33+
SHA=`tail $ROOT_TEST_DIR/testdata/mobydick.txt | shasum | awk '{ print $1 }'`
34+
assert_equal $SHA `shasum < $BATS_TMPDIR/mobydick_test.txt | awk '{ print $1 }'`
35+
else
36+
skip
37+
fi
38+
}
39+
40+
@test "te: key not available" {
41+
if [ "$TRANSPARENT_ENCRYPTION" = "true" ]; then
42+
run $HADOOP_FS -mkdir -p /_test/kms-no-key
43+
assert_success
44+
run $HADOOP_KEY create key-removed
45+
assert_success
46+
run hdfs crypto -createZone -keyName key-removed -path /_test/kms-no-key
47+
assert_success
48+
run $HADOOP_FS -put $ROOT_TEST_DIR/testdata/foo.txt /_test/kms-no-key/foo
49+
assert_success
50+
run $HADOOP_KEY delete key-removed -f
51+
assert_success
52+
run $HDFS cat /_test/kms-no-key/foo
53+
assert_failure
54+
assert_output "open /_test/kms-no-key/foo: kms: 'key-removed@0' not found"
55+
56+
run $HDFS put $ROOT_TEST_DIR/testdata/foo.txt /_test/kms-no-key/foo2
57+
assert_failure
58+
assert_output "create /_test/kms-no-key/foo2: kms: 'key-removed@0' not found"
59+
60+
run $HDFS ls /_test/kms-no-key/foo2
61+
assert_failure
62+
assert_output "stat /_test/kms-no-key/foo2: file does not exist"
63+
else
64+
skip
65+
fi
66+
}

‎file_writer_test.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -531,3 +531,83 @@ func TestFileAppendDeadlineBefore(t *testing.T) {
531531
_, err = writer.Write([]byte("foo\n"))
532532
assert.Error(t, err)
533533
}
534+
535+
func skipWithoutEncryptedZone(t *testing.T) {
536+
if os.Getenv("TRANSPARENT_ENCRYPTION") != "true" {
537+
t.Skip("Skipping, this test requires encryption zone to make sense")
538+
}
539+
}
540+
541+
func TestEncryptedZoneWriteChunks(t *testing.T) {
542+
skipWithoutEncryptedZone(t)
543+
544+
originalText := []byte("some random plain text, nice to have it quite long")
545+
client := getClient(t)
546+
writer, err := client.Create("/_test/kms/write_chunks.txt")
547+
require.NoError(t, err)
548+
549+
var pos int64 = 0
550+
for _, x := range []int{5, 7, 6, 4, 28} {
551+
_, err = writer.Write(originalText[pos : pos+int64(x)])
552+
require.NoError(t, err)
553+
pos += int64(x)
554+
}
555+
assertClose(t, writer)
556+
557+
reader, err := client.Open("/_test/kms/write_chunks.txt")
558+
require.NoError(t, err)
559+
560+
bytes, err := ioutil.ReadAll(reader)
561+
require.NoError(t, err)
562+
assert.Equal(t, originalText, bytes)
563+
}
564+
565+
func TestEncryptedZoneAppendChunks(t *testing.T) {
566+
skipWithoutEncryptedZone(t)
567+
568+
originalText := []byte("some random plain text, nice to have it quite long")
569+
client := getClient(t)
570+
writer, err := client.Create("/_test/kms/append_chunks.txt")
571+
require.NoError(t, err)
572+
assertClose(t, writer)
573+
574+
var pos int64 = 0
575+
for _, x := range []int{5, 7, 6, 4, 28} {
576+
writer, err := client.Append("/_test/kms/append_chunks.txt")
577+
require.NoError(t, err)
578+
_, err = writer.Write(originalText[pos : pos+int64(x)])
579+
require.NoError(t, err)
580+
pos += int64(x)
581+
assertClose(t, writer)
582+
}
583+
584+
reader, err := client.Open("/_test/kms/append_chunks.txt")
585+
require.NoError(t, err)
586+
bytes, err := ioutil.ReadAll(reader)
587+
require.NoError(t, err)
588+
assert.Equal(t, originalText, bytes)
589+
}
590+
591+
func TestEncryptedZoneLargeBlock(t *testing.T) {
592+
skipWithoutEncryptedZone(t)
593+
594+
// Generate quite large (aesChunkSize * 1.5 bytes) block, so we can trigger encryption in chunks.
595+
str := "some random text"
596+
originalText := []byte(strings.Repeat(str, aesChunkSize*1.5/len(str)))
597+
client := getClient(t)
598+
599+
// Create file with small (128Kb) block size, so encrypted chunk will be placed over multiple hdfs blocks.
600+
writer, err := client.CreateFile("/_test/kms/large_write.txt", 1, 131072, 0755)
601+
require.NoError(t, err)
602+
603+
_, err = writer.Write(originalText)
604+
require.NoError(t, err)
605+
assertClose(t, writer)
606+
607+
reader, err := client.Open("/_test/kms/large_write.txt")
608+
require.NoError(t, err)
609+
610+
bytes, err := ioutil.ReadAll(reader)
611+
require.NoError(t, err)
612+
assert.Equal(t, originalText, bytes)
613+
}

0 commit comments

Comments
 (0)