johanels/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-12-11 22:14:05 +01:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

hdfs: support kerberos authentication #42

Browse Source
This commit is contained in:
Yury Stankevich
2021-01-16 18:52:08 +03:00
committed by Nick Craig-Wood
parent df4e6079f1
commit b569dc11a0
13 changed files with 269 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fstest/testserver/images/test-hdfs/Dockerfile
View File

@@ -3,12 +3,11 @@ FROM debian:stretch
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-8-jdk \
net-tools curl python krb5-user krb5-kdc krb5-admin-server \
&& rm -rf /var/lib/apt/lists/*
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends net-tools curl python
ENV HADOOP_VERSION 3.2.1
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
RUN set -x \
@@ -37,6 +36,10 @@ ADD kms-site.xml /etc/hadoop/kms-site.xml
ADD mapred-site.xml /etc/hadoop/mapred-site.xml
ADD yarn-site.xml /etc/hadoop/yarn-site.xml
ADD krb5.conf /etc/
ADD kdc.conf /etc/krb5kdc/
RUN echo '*/admin@KERBEROS.RCLONE *' > /etc/krb5kdc/kadm5.acl
ADD run.sh /run.sh
RUN chmod a+x /run.sh
CMD ["/run.sh"]
CMD ["/run.sh"]

33
fstest/testserver/images/test-hdfs/README.md
View File

@@ -26,7 +26,32 @@ cd backend/hdfs
GO111MODULE=on go test -v
```
stop docker image:
```
docker kill rclone-hdfs
```
hdfs logs will be available in `.stdout.log` and `.stderr.log`
# Kerberos
test can be run against kerberos-enabled hdfs
1. configure local krb5.conf
```
[libdefaults]
default_realm = KERBEROS.RCLONE
[realms]
KERBEROS.RCLONE = {
kdc = localhost
}
```
2. enable kerberos in remote configuration
```
[TestHdfs]
...
service_principal_name = hdfs/localhost
data_transfer_protection = privacy
```
3. run test
```
cd backend/hdfs
KERBEROS=true GO111MODULE=on go test -v
```

6
fstest/testserver/images/test-hdfs/core-site.xml
View File

@@ -3,4 +3,10 @@
<property><name>hadoop.http.staticuser.user</name><value>root</value></property>
<property><name>hadoop.proxyuser.root.groups</name><value>root,nogroup</value></property>
<property><name>hadoop.proxyuser.root.hosts</name><value>*</value></property>
<!-- KERBEROS BEGIN -->
<property><name>hadoop.security.authentication</name><value>kerberos</value></property>
<property><name>hadoop.security.authorization</name><value>true</value></property>
<property><name>hadoop.rpc.protection</name><value>integrity</value></property>
<property><name>hadoop.user.group.static.mapping.overrides</name><value>user=supergroup</value></property>
<!-- KERBEROS END -->
</configuration>

17
fstest/testserver/images/test-hdfs/hdfs-site.xml
View File

@@ -11,4 +11,21 @@
<property><name>dfs.namenode.servicerpc-bind-host</name><value>0.0.0.0</value></property>
<property><name>dfs.replication</name><value>2</value></property>
<property><name>nfs.dump.dir</name><value>/tmp</value></property>
<!-- KERBEROS BEGIN -->
<property><name>ignore.secure.ports.for.testing</name><value>true</value></property>
<property><name>dfs.safemode.extension</name><value>0</value></property>
<property><name>dfs.block.access.token.enable</name><value>true</value></property>
<property><name>dfs.encrypt.data.transfer</name><value>true</value></property>
<property><name>dfs.encrypt.data.transfer.algorithm</name><value>rc4</value></property>
<property><name>dfs.encrypt.data.transfer.cipher.suites</name><value>AES/CTR/NoPadding</value></property>
<property><name>dfs.namenode.kerberos.principal</name> <value>hdfs/_HOST@KERBEROS.RCLONE</value></property>
<property><name>dfs.web.authentication.kerberos.principal</name><value>HTTP/_HOST@KERBEROS.RCLONE</value></property>
<property><name>dfs.datanode.kerberos.principal</name> <value>hdfs/_HOST@KERBEROS.RCLONE</value></property>
<property><name>dfs.namenode.keytab.file</name> <value>/etc/hadoop/kerberos.key</value></property>
<property><name>dfs.web.authentication.kerberos.keytab</name><value>/etc/hadoop/kerberos.key</value></property>
<property><name>dfs.datanode.keytab.file</name> <value>/etc/hadoop/kerberos.key</value></property>
<!-- KERBEROS END -->
</configuration>

4
fstest/testserver/images/test-hdfs/kdc.conf Normal file
View File

@@ -0,0 +1,4 @@
[realms]
KERBEROS.RCLONE = {
acl_file = /etc/krb5kdc/kadm5.acl
}

10
fstest/testserver/images/test-hdfs/krb5.conf Normal file
View File

@@ -0,0 +1,10 @@
[libdefaults]
default_realm = KERBEROS.RCLONE
dns_lookup_realm = false
dns_lookup_kdc = false
forwardable = true
proxiable = true
[realms]
KERBEROS.RCLONE = {
kdc = localhost
}

25
fstest/testserver/images/test-hdfs/run.sh
View File

@@ -1,5 +1,30 @@
#!/bin/bash
KERBEROS=${KERBEROS-"false"}
if [ $KERBEROS = "true" ]; then
echo prepare kerberos
ADMIN_PASSWORD="kerberos"
USER_PASSWORD="user"
echo -e "$ADMIN_PASSWORD\n$ADMIN_PASSWORD" | kdb5_util -r "KERBEROS.RCLONE" create -s
echo -e "$ADMIN_PASSWORD\n$ADMIN_PASSWORD" | kadmin.local -q "addprinc hadoop/admin"
echo -e "$USER_PASSWORD\n$USER_PASSWORD" | kadmin.local -q "addprinc user"
kadmin.local -q 'addprinc -randkey hdfs/localhost'
kadmin.local -q 'addprinc -randkey hdfs/rclone-hdfs'
kadmin.local -q 'addprinc -randkey HTTP/localhost'
kadmin.local -p hadoop/admin -q "ktadd -k /etc/hadoop/kerberos.key hdfs/localhost hdfs/rclone-hdfs HTTP/localhost"
service krb5-kdc restart
echo -e "$USER_PASSWORD\n" | kinit user
klist
echo kerberos ready
else
echo drop kerberos from configuration files
sed -i '/KERBEROS BEGIN/,/KERBEROS END/d' /etc/hadoop/core-site.xml
sed -i '/KERBEROS BEGIN/,/KERBEROS END/d' /etc/hadoop/hdfs-site.xml
fi
echo format namenode
hdfs namenode -format test

15
fstest/testserver/init.d/TestHdfs
View File

@@ -3,19 +3,32 @@
set -e
NAME=rclone-hdfs
KERBEROS=${KERBEROS-"false"}
. $(dirname "$0")/docker.bash
start() {
docker run --rm -d --name "rclone-hdfs" -p 127.0.0.1:9866:9866 -p 127.0.0.1:8020:8020 --hostname "rclone-hdfs" rclone/test-hdfs
docker run --rm -d --name "rclone-hdfs" \
--hostname "rclone-hdfs" \
-e "KERBEROS=$KERBEROS" \
-p 127.0.0.1:9866:9866 \
-p 127.0.0.1:8020:8020 \
-p 127.0.0.1:750:750 \
-p 127.0.0.1:88:88 \
rclone/test-hdfs
sleep 10
if [ $KERBEROS = "true" ]; then
docker cp rclone-hdfs:/tmp/krb5cc_0 /tmp/krb5cc_`id -u`
fi
echo type=hdfs
echo namenode=127.0.0.1:8020
echo username=root
}
stop() {
if status ; then
docker logs $NAME > .stdout.log 2> .stderr.log
docker kill $NAME
echo "$NAME stopped"
fi