From 6cd72c3b0e58b2fdd9a78ff6816b8933f49eb53a Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Fri, 5 Sep 2025 21:55:12 -0700 Subject: [PATCH 1/5] RANGER-5310:Include Apache Tez as the process framework for ranger-hive docker Signed-off-by: Ramesh Mani --- dev-support/ranger-docker/.env | 4 +- dev-support/ranger-docker/Dockerfile.ranger | 3 + .../ranger-docker/Dockerfile.ranger-hadoop | 7 + .../ranger-docker/Dockerfile.ranger-hbase | 5 +- .../ranger-docker/Dockerfile.ranger-hive | 17 ++- .../ranger-docker/Dockerfile.ranger-kafka | 6 +- .../ranger-docker/Dockerfile.ranger-kms | 4 +- .../ranger-docker/Dockerfile.ranger-knox | 5 +- .../ranger-docker/Dockerfile.ranger-tagsync | 5 +- .../ranger-docker/Dockerfile.ranger-usersync | 5 +- .../docker-compose.ranger-hadoop.yml | 2 + .../docker-compose.ranger-hive.yml | 3 + .../ranger-docker/download-archives.sh | 2 + .../ranger-docker/scripts/create-users.sh | 62 ++++++++ .../scripts/hive-site-metastore-mysql.xml | 132 ++++++++++++++++++ .../ranger-docker/scripts/hive-site-mysql.xml | 68 +++++++++ .../scripts/hive-site-oracle.xml | 68 +++++++++ .../scripts/hive-site-postgres.xml | 68 +++++++++ .../scripts/hive-site-sqlserver.xml | 68 +++++++++ .../scripts/ranger-hadoop-setup.sh | 65 +++++++++ .../scripts/ranger-hive-setup.sh | 128 +++++++++++++++++ .../ranger-docker/scripts/tez-site.xml | 93 ++++++++++++ 22 files changed, 809 insertions(+), 11 deletions(-) create mode 100755 dev-support/ranger-docker/scripts/create-users.sh create mode 100644 dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml create mode 100644 dev-support/ranger-docker/scripts/tez-site.xml diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index e6de538a23..e65b83627b 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -25,8 +25,8 @@ SOLR_VERSION=8.11.3 HADOOP_VERSION=3.3.6 HBASE_VERSION=2.6.0 HIVE_VERSION=4.0.1 -HIVE_HADOOP_VERSION=3.1.1 -KAFKA_VERSION=2.8.2 +HIVE_HADOOP_VERSION=3.3.6 +TEZ_VERSION=0.10.4 KNOX_VERSION=2.0.0 TRINO_VERSION=377 OZONE_VERSION=1.4.0 diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index b6dcff30a4..fecc7b2655 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -29,6 +29,7 @@ COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/ COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-admin-install.properties COPY ./scripts/create-ranger-services.py ${RANGER_SCRIPTS}/ +COPY ./scripts/create-users.sh ${RANGER_SCRIPTS}/ RUN tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} \ && ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin \ @@ -37,6 +38,8 @@ RUN tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --direct && mkdir -p /var/run/ranger \ && mkdir -p /var/log/ranger \ && chown -R ranger:ranger ${RANGER_HOME}/admin/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ \ + && chmod +x ${RANGER_SCRIPTS}/create-users.sh \ + && ${RANGER_SCRIPTS}/create-users.sh \ && chmod 755 ${RANGER_SCRIPTS}/ranger.sh \ && mkdir -p /usr/share/java/ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop index 5fc455e4b5..3a5761f03c 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop +++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop @@ -21,22 +21,27 @@ FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG HADOOP_VERSION ARG HDFS_PLUGIN_VERSION ARG YARN_PLUGIN_VERSION +ARG TEZ_VERSION COPY ./dist/version /home/ranger/dist/ COPY ./dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/ COPY ./dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz /home/ranger/dist/ COPY ./downloads/hadoop-${HADOOP_VERSION}.tar.gz /home/ranger/dist/ +COPY ./downloads/apache-tez-${TEZ_VERSION}-bin.tar.gz /home/ranger/dist/ COPY ./scripts/ranger-hadoop-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-hadoop.sh /home/ranger/scripts/ COPY ./scripts/ranger-hadoop-mkdir.sh /home/ranger/scripts/ COPY ./scripts/ranger-hdfs-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/ranger-yarn-plugin-install.properties /home/ranger/scripts/ +COPY ./scripts/create-users.sh /home/ranger/scripts/ RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \ ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \ rm -f /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz && \ + tar xvfz /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz --directory=/opt/ && \ + ln -s /opt/apache-tez-${TEZ_VERSION}-bin /opt/tez && \ tar xvfz /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz --directory=/opt/ranger && \ ln -s /opt/ranger/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin /opt/ranger/ranger-hdfs-plugin && \ rm -f /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz && \ @@ -46,6 +51,8 @@ RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ rm -f /home/ranger/dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-yarn-plugin-install.properties /opt/ranger/ranger-yarn-plugin/install.properties && \ chmod 744 ${RANGER_SCRIPTS}/ranger-hadoop-setup.sh ${RANGER_SCRIPTS}/ranger-hadoop.sh ${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh && \ + chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ + ${RANGER_SCRIPTS}/create-users.sh && \ chown hdfs:hadoop ${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh RUN apt-get update && \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase index 0d01200796..0d1884f0ab 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hbase +++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase @@ -29,6 +29,7 @@ COPY ./downloads/hbase-${HBASE_VERSION}-bin.tar.gz /home/ranger/dis COPY ./scripts/ranger-hbase-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/ COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/ +COPY ./scripts/create-users.sh /home/ranger/scripts/ COPY ./scripts/hbase-site.xml /home/ranger/scripts/ RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \ @@ -38,7 +39,9 @@ RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/op ln -s /opt/ranger/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin /opt/ranger/ranger-hbase-plugin && \ rm -f /home/ranger/dist/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-hbase-plugin-install.properties /opt/ranger/ranger-hbase-plugin/install.properties && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-hbase-setup.sh ${RANGER_SCRIPTS}/ranger-hbase.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-hbase-setup.sh ${RANGER_SCRIPTS}/ranger-hbase.sh && \ + chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ + ${RANGER_SCRIPTS}/create-users.sh RUN apt-get update && \ apt-get install -y --no-install-recommends openssh-server && \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive index 7fddfc0019..c52e252ff3 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hive +++ b/dev-support/ranger-docker/Dockerfile.ranger-hive @@ -22,6 +22,7 @@ FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG HIVE_VERSION ARG HIVE_HADOOP_VERSION ARG HIVE_PLUGIN_VERSION +ARG TEZ_VERSION ARG RANGER_DB_TYPE @@ -29,6 +30,7 @@ COPY ./dist/version /home/ranger/dist/ COPY ./dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz /home/ranger/dist/ COPY ./downloads/apache-hive-${HIVE_VERSION}-bin.tar.gz /home/ranger/dist/ COPY ./downloads/hadoop-${HIVE_HADOOP_VERSION}.tar.gz /home/ranger/dist/ +COPY ./downloads/apache-tez-${TEZ_VERSION}-bin.tar.gz /home/ranger/dist/ COPY ./downloads/postgresql-42.2.16.jre7.jar /home/ranger/dist/ COPY ./downloads/mysql-connector-java-8.0.28.jar /home/ranger/dist/ COPY ./downloads/ojdbc8.jar /home/ranger/dist/ @@ -37,8 +39,10 @@ COPY ./scripts/ranger-hive-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-hive.sh /home/ranger/scripts/ COPY ./scripts/ranger-hive-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/hive-site-${RANGER_DB_TYPE}.xml /home/ranger/scripts/hive-site.xml +COPY ./scripts/hive-site-metastore-${RANGER_DB_TYPE}.xml /home/ranger/scripts/hive-site-metastore.xml +COPY ./scripts/create-users.sh /home/ranger/scripts/ -RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \ +RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \ rm -f /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ mv /home/ranger/dist/postgresql-42.2.16.jre7.jar /opt/hive/lib/ && \ @@ -47,15 +51,22 @@ RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --director tar xvfz /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --directory=/opt/ && \ ln -s /opt/hadoop-${HIVE_HADOOP_VERSION} /opt/hadoop && \ rm -f /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \ + tar xvfz /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz --directory=/opt/ && \ + ln -s /opt/apache-tez-${TEZ_VERSION}-bin /opt/tez && \ + rm -f /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz && \ tar xvfz /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \ ln -s /opt/ranger/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \ rm -f /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-hive-plugin-install.properties /opt/ranger/ranger-hive-plugin/install.properties && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-hive-setup.sh ${RANGER_SCRIPTS}/ranger-hive.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-hive-setup.sh ${RANGER_SCRIPTS}/ranger-hive.sh && \ + chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ + ${RANGER_SCRIPTS}/create-users.sh ENV HIVE_HOME=/opt/hive ENV HADOOP_HOME=/opt/hadoop -ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin +ENV TEZ_HOME=/opt/tez +ENV TEZ_CONF_DIR=/opt/tez/conf +ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin:/opt/tez/bin ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka index 48c5789a7b..fdc8258e56 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kafka +++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka @@ -29,6 +29,7 @@ COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/ranger/dist COPY ./scripts/ranger-kafka-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-kafka.sh /home/ranger/scripts/ COPY ./scripts/ranger-kafka-plugin-install.properties /home/ranger/scripts/ +COPY ./scripts/create-users.sh /home/ranger/scripts/ RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \ ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \ @@ -37,7 +38,10 @@ RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ ln -s /opt/ranger/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin /opt/ranger/ranger-kafka-plugin && \ rm -f /home/ranger/dist/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-kafka-plugin-install.properties /opt/ranger/ranger-kafka-plugin/install.properties && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-kafka-setup.sh ${RANGER_SCRIPTS}/ranger-kafka.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-kafka-setup.sh ${RANGER_SCRIPTS}/ranger-kafka.sh && \ + chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ + ${RANGER_SCRIPTS}/create-users.sh + ENV KAFKA_HOME=/opt/kafka ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms index 55401ef60e..e1d97c650b 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kms +++ b/dev-support/ranger-docker/Dockerfile.ranger-kms @@ -26,6 +26,7 @@ COPY ./dist/ranger-${KMS_VERSION}-kms.tar.gz /home/ranger/dist COPY ./scripts/ranger-kms.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-kms-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-kms-install.properties +COPY ./scripts/create-users.sh /home/ranger/scripts/ RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${KMS_VERSION}-kms ${RANGER_HOME}/kms && \ @@ -39,7 +40,8 @@ RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RA ln -s /etc/init.d/ranger-kms /etc/rc3.d/K90ranger-kms && \ ln -s ${RANGER_HOME}/kms/ranger-kms-services.sh /usr/bin/ranger-kms-services.sh && \ chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-kms.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-kms.sh && \ + ${RANGER_SCRIPTS}/create-users.sh FROM ranger-kms AS ranger_postgres COPY ./downloads/postgresql-42.2.16.jre7.jar /home/ranger/dist/ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-knox b/dev-support/ranger-docker/Dockerfile.ranger-knox index 653af09ee2..13bbf006aa 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-knox +++ b/dev-support/ranger-docker/Dockerfile.ranger-knox @@ -25,6 +25,7 @@ ARG KNOX_PLUGIN_VERSION COPY ./dist/version /home/ranger/dist/ COPY ./dist/ranger-${KNOX_PLUGIN_VERSION}-knox-plugin.tar.gz /home/ranger/dist/ COPY ./downloads/knox-${KNOX_VERSION}.tar.gz /home/ranger/dist/ +COPY ./scripts/create-users.sh /home/ranger/scripts/ COPY ./scripts/ranger-knox-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-knox.sh /home/ranger/scripts/ @@ -40,7 +41,9 @@ RUN tar xvfz /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz --directory=/opt/ && rm -f /home/ranger/dist/ranger-${KNOX_PLUGIN_VERSION}-knox-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-knox-plugin-install.properties /opt/ranger/ranger-knox-plugin/install.properties && \ cp -f /home/ranger/scripts/ranger-knox-sandbox.xml /opt/knox/conf/topologies/sandbox.xml && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-knox-setup.sh ${RANGER_SCRIPTS}/ranger-knox.sh ${RANGER_SCRIPTS}/ranger-knox-expect.py + chmod 744 ${RANGER_SCRIPTS}/ranger-knox-setup.sh ${RANGER_SCRIPTS}/ranger-knox.sh ${RANGER_SCRIPTS}/ranger-knox-expect.py && \ + chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ + ${RANGER_SCRIPTS}/create-users.sh ENV KNOX_HOME=/opt/knox ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/knox/bin diff --git a/dev-support/ranger-docker/Dockerfile.ranger-tagsync b/dev-support/ranger-docker/Dockerfile.ranger-tagsync index 59efb40eb3..5b0573252b 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-tagsync +++ b/dev-support/ranger-docker/Dockerfile.ranger-tagsync @@ -26,6 +26,7 @@ COPY ./dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz /home/ranger/dist/ COPY ./scripts/ranger-tagsync.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-tagsync-install.properties ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-tagsync-tags.json ${RANGER_SCRIPTS}/ +COPY ./scripts/create-users.sh ${RANGER_SCRIPTS}/ RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${TAGSYNC_VERSION}-tagsync ${RANGER_HOME}/tagsync && \ @@ -43,7 +44,9 @@ RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --direct ln -s /etc/init.d/ranger-tagsync /etc/rc3.d/K00ranger-tagsync && \ ln -s ${RANGER_HOME}/tagsync/ranger-tagsync-services.sh /usr/bin/ranger-tagsync-services.sh && \ chown -R ranger:ranger ${RANGER_HOME}/tagsync/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ /etc/ranger /etc/init.d/ranger-tagsync && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-tagsync.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-tagsync.sh && \ + chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ + ${RANGER_SCRIPTS}/create-users.sh USER ranger diff --git a/dev-support/ranger-docker/Dockerfile.ranger-usersync b/dev-support/ranger-docker/Dockerfile.ranger-usersync index 9b164cad06..893d440578 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-usersync +++ b/dev-support/ranger-docker/Dockerfile.ranger-usersync @@ -26,6 +26,7 @@ COPY ./dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz /home/ranger/dist/ COPY ./scripts/ranger-usersync.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-usersync-install.properties ${RANGER_SCRIPTS}/ COPY ./scripts/ugsync-file-source.csv ${RANGER_SCRIPTS}/ +COPY ./scripts/create-users.sh ${RANGER_SCRIPTS}/ RUN tar xvfz /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${USERSYNC_VERSION}-usersync ${RANGER_HOME}/usersync && \ @@ -42,7 +43,9 @@ RUN tar xvfz /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz --dire ln -s /etc/init.d/ranger-usersync /etc/rc3.d/K00ranger-usersync && \ ln -s ${RANGER_HOME}/usersync/ranger-usersync-services.sh /usr/bin/ranger-usersync && \ chown -R ranger:ranger ${RANGER_HOME}/usersync/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ /etc/ranger /etc/init.d/ranger-usersync && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-usersync.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-usersync.sh && \ + chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ + ${RANGER_SCRIPTS}/create-users.sh USER ranger diff --git a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml index dacbbf0ccc..37fe6b5d83 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml @@ -9,6 +9,7 @@ services: - HADOOP_VERSION=${HADOOP_VERSION} - HDFS_PLUGIN_VERSION=${HDFS_PLUGIN_VERSION} - YARN_PLUGIN_VERSION=${YARN_PLUGIN_VERSION} + - TEZ_VERSION=${TEZ_VERSION} image: ranger-hadoop container_name: ranger-hadoop hostname: ranger-hadoop.example.com @@ -32,6 +33,7 @@ services: - HADOOP_VERSION - HDFS_PLUGIN_VERSION - YARN_PLUGIN_VERSION + - TEZ_VERSION networks: ranger: diff --git a/dev-support/ranger-docker/docker-compose.ranger-hive.yml b/dev-support/ranger-docker/docker-compose.ranger-hive.yml index 5815a472d6..6084630825 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-hive.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-hive.yml @@ -9,6 +9,7 @@ services: - HIVE_HADOOP_VERSION=${HIVE_HADOOP_VERSION} - HIVE_VERSION=${HIVE_VERSION} - HIVE_PLUGIN_VERSION=${HIVE_PLUGIN_VERSION} + - TEZ_VERSION=${TEZ_VERSION} - RANGER_DB_TYPE=${RANGER_DB_TYPE} image: ranger-hive container_name: ranger-hive @@ -19,6 +20,7 @@ services: - ranger ports: - "10000:10000" + - "9083:9083" depends_on: ranger: condition: service_started @@ -30,6 +32,7 @@ services: - HIVE_HADOOP_VERSION - HIVE_VERSION - HIVE_PLUGIN_VERSION + - TEZ_VERSION - RANGER_DB_TYPE networks: diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh index 0bfca41ade..1ce4240e4d 100755 --- a/dev-support/ranger-docker/download-archives.sh +++ b/dev-support/ranger-docker/download-archives.sh @@ -53,6 +53,7 @@ then downloadIfNotPresent hbase-${HBASE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hbase/${HBASE_VERSION} downloadIfNotPresent apache-hive-${HIVE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hive/hive-${HIVE_VERSION} downloadIfNotPresent hadoop-${HIVE_HADOOP_VERSION}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION} + downloadIfNotPresent apache-tez-${TEZ_VERSION}-bin.tar.gz https://archive.apache.org/dist/tez/${TEZ_VERSION} downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz https://archive.apache.org/dist/kafka/${KAFKA_VERSION} downloadIfNotPresent knox-${KNOX_VERSION}.tar.gz https://archive.apache.org/dist/knox/${KNOX_VERSION} downloadIfNotPresent ozone-${OZONE_VERSION}.tar.gz https://archive.apache.org/dist/ozone/${OZONE_VERSION} @@ -72,6 +73,7 @@ else then downloadIfNotPresent apache-hive-${HIVE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hive/hive-${HIVE_VERSION} downloadIfNotPresent hadoop-${HIVE_HADOOP_VERSION}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION} + downloadIfNotPresent apache-tez-${TEZ_VERSION}-bin.tar.gz https://archive.apache.org/dist/tez/${TEZ_VERSION} elif [[ $arg == 'kafka' ]] then downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz https://archive.apache.org/dist/kafka/${KAFKA_VERSION} diff --git a/dev-support/ranger-docker/scripts/create-users.sh b/dev-support/ranger-docker/scripts/create-users.sh new file mode 100755 index 0000000000..dddca21e24 --- /dev/null +++ b/dev-support/ranger-docker/scripts/create-users.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to create alice and abram users in ranger containers +# This script is designed to be run during container initialization + +# Function to create a user for testing. +create_user_if_not_exists() { + local username=$1 + local uid=$2 + local gid=$3 + local home_dir=$4 + + if ! id "$username" &>/dev/null; then + echo "Creating user: $username (uid:$uid, gid:$gid)" + useradd -u "$uid" -g "$gid" -m -d "$home_dir" -s /bin/bash "$username" + + # Set a default password (same as username for demo purposes) + echo "$username:$username" | chpasswd + + # Add user to hadoop group for HDFS access + if getent group hadoop &>/dev/null; then + usermod -a -G hadoop "$username" + fi + + # Create .ssh directory and set proper permissions + mkdir -p "$home_dir/.ssh" + chmod 700 "$home_dir/.ssh" + chown "$username:$gid" "$home_dir/.ssh" + + echo "User $username created successfully" + else + echo "User $username already exists" + fi +} + +# Ensure hadoop group exists (gid 1001 is used by hdfs, yarn, hive users) +if ! getent group hadoop &>/dev/null; then + groupadd -g 1001 hadoop + echo "Created hadoop group" +fi + +# Create alice user (uid: 2001, gid: 1001 - hadoop group) +create_user_if_not_exists "alice" 2001 1001 "/home/alice" + +# Create abram user (uid: 2002, gid: 1001 - hadoop group) +create_user_if_not_exists "abram" 2002 1001 "/home/abram" diff --git a/dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml b/dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml new file mode 100644 index 0000000000..1049239f58 --- /dev/null +++ b/dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml @@ -0,0 +1,132 @@ + + + + + + javax.jdo.option.ConnectionURL + jdbc:mysql://ranger-db/hive + + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + + + javax.jdo.option.ConnectionUserName + hive + + + javax.jdo.option.ConnectionPassword + rangerR0cks! + + + + + hive.server2.enable.doAs + false + + + + + hive.metastore.warehouse.dir + file:///warehouse/tablespace/managed/hive + + + hive.metastore.warehouse.external.dir + file:///warehouse/tablespace/external/hive + + + + hive.zookeeper.quorum + ranger-zk.example.com + + + hive.zookeeper.client.port + 2181 + + + + + hive.execution.engine + tez + Execution engine to use for Hive queries + + + + hive.tez.container.size + 1024 + By default Tez will spawn containers of the size of a mapper + + + + hive.tez.java.opts + -Xmx768m + Java command line options for Tez + + + + hive.tez.log.level + INFO + Log level for Tez + + + + hive.prewarm.enabled + false + Enables container prewarm for Tez + + + + hive.prewarm.numcontainers + 3 + Controls the number of containers to prewarm for Tez + + + + hive.tez.auto.reducer.parallelism + true + Turn on Tez' auto reducer parallelism feature + + + + hive.tez.min.reducer.per.query + 2 + Controls the minimum number of reducers for Tez + + + + hive.tez.max.reducer.per.query + 999 + Controls the maximum number of reducers for Tez + + + + + hive.tez.exec.print.summary + true + Whether to print a summary of the plan execution + + + + tez.queue.name + default + YARN queue name for Tez jobs + + diff --git a/dev-support/ranger-docker/scripts/hive-site-mysql.xml b/dev-support/ranger-docker/scripts/hive-site-mysql.xml index 118cdbbca9..ccb095329b 100644 --- a/dev-support/ranger-docker/scripts/hive-site-mysql.xml +++ b/dev-support/ranger-docker/scripts/hive-site-mysql.xml @@ -49,4 +49,72 @@ hive.zookeeper.client.port 2181 + + + + hive.execution.engine + tez + Execution engine to use for Hive queries + + + + hive.tez.container.size + 1024 + By default Tez will spawn containers of the size of a mapper + + + + hive.tez.java.opts + -Xmx768m + Java command line options for Tez + + + + hive.tez.log.level + INFO + Log level for Tez + + + + hive.prewarm.enabled + false + Enables container prewarm for Tez + + + + hive.prewarm.numcontainers + 3 + Controls the number of containers to prewarm for Tez + + + + hive.tez.auto.reducer.parallelism + true + Turn on Tez' auto reducer parallelism feature + + + + hive.tez.min.reducer.per.query + 2 + Controls the minimum number of reducers for Tez + + + + hive.tez.max.reducer.per.query + 999 + Controls the maximum number of reducers for Tez + + + + + hive.tez.exec.print.summary + true + Whether to print a summary of the plan execution + + + + tez.queue.name + default + YARN queue name for Tez jobs + diff --git a/dev-support/ranger-docker/scripts/hive-site-oracle.xml b/dev-support/ranger-docker/scripts/hive-site-oracle.xml index 2b8cc5e08f..53cde12799 100644 --- a/dev-support/ranger-docker/scripts/hive-site-oracle.xml +++ b/dev-support/ranger-docker/scripts/hive-site-oracle.xml @@ -49,4 +49,72 @@ hive.zookeeper.client.port 2181 + + + + hive.execution.engine + tez + Execution engine to use for Hive queries + + + + hive.tez.container.size + 1024 + By default Tez will spawn containers of the size of a mapper + + + + hive.tez.java.opts + -Xmx768m + Java command line options for Tez + + + + hive.tez.log.level + INFO + Log level for Tez + + + + hive.prewarm.enabled + false + Enables container prewarm for Tez + + + + hive.prewarm.numcontainers + 3 + Controls the number of containers to prewarm for Tez + + + + hive.tez.auto.reducer.parallelism + true + Turn on Tez' auto reducer parallelism feature + + + + hive.tez.min.reducer.per.query + 2 + Controls the minimum number of reducers for Tez + + + + hive.tez.max.reducer.per.query + 999 + Controls the maximum number of reducers for Tez + + + + + hive.tez.exec.print.summary + true + Whether to print a summary of the plan execution + + + + tez.queue.name + default + YARN configuration for Tez jobs + diff --git a/dev-support/ranger-docker/scripts/hive-site-postgres.xml b/dev-support/ranger-docker/scripts/hive-site-postgres.xml index 55343a3234..f139af5d42 100644 --- a/dev-support/ranger-docker/scripts/hive-site-postgres.xml +++ b/dev-support/ranger-docker/scripts/hive-site-postgres.xml @@ -49,4 +49,72 @@ hive.zookeeper.client.port 2181 + + + + hive.execution.engine + tez + Execution engine to use for Hive queries + + + + hive.tez.container.size + 1024 + By default Tez will spawn containers of the size of a mapper + + + + hive.tez.java.opts + -Xmx768m + Java command line options for Tez + + + + hive.tez.log.level + INFO + Log level for Tez + + + + hive.prewarm.enabled + false + Enables container prewarm for Tez + + + + hive.prewarm.numcontainers + 3 + Controls the number of containers to prewarm for Tez + + + + hive.tez.auto.reducer.parallelism + true + Turn on Tez' auto reducer parallelism feature + + + + hive.tez.min.reducer.per.query + 2 + Controls the minimum number of reducers for Tez + + + + hive.tez.max.reducer.per.query + 999 + Controls the maximum number of reducers for Tez + + + + + hive.tez.exec.print.summary + true + Whether to print a summary of the plan execution + + + + tez.queue.name + default + YARN queue name for Tez jobs + diff --git a/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml index 5bed21c2d8..2298f4b644 100644 --- a/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml +++ b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml @@ -47,4 +47,72 @@ hive.zookeeper.client.port 2181 + + + + hive.execution.engine + tez + Execution engine to use for Hive queries + + + + hive.tez.container.size + 1024 + By default Tez will spawn containers of the size of a mapper + + + + hive.tez.java.opts + -Xmx768m + Java command line options for Tez + + + + hive.tez.log.level + INFO + Log level for Tez + + + + hive.prewarm.enabled + false + Enables container prewarm for Tez + + + + hive.prewarm.numcontainers + 3 + Controls the number of containers to prewarm for Tez + + + + hive.tez.auto.reducer.parallelism + true + Turn on Tez' auto reducer parallelism feature + + + + hive.tez.min.reducer.per.query + 2 + Controls the minimum number of reducers for Tez + + + + hive.tez.max.reducer.per.query + 999 + Controls the maximum number of reducers for Tez + + + + + hive.tez.exec.print.summary + true + Whether to print a summary of the plan execution + + + + tez.queue.name + default + YARN queue name for Tez jobs + diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh index 10f04acd9f..d87548b80f 100755 --- a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh +++ b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh @@ -52,16 +52,81 @@ cat < ${HADOOP_HOME}/etc/hadoop/yarn-site.xml yarn.nodemanager.aux-services mapreduce_shuffle + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + yarn.nodemanager.env-whitelist JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME + + yarn.resourcemanager.hostname + ranger-hadoop + + + yarn.nodemanager.resource.memory-mb + 4096 + + + yarn.scheduler.maximum-allocation-mb + 4096 + + + yarn.scheduler.minimum-allocation-mb + 256 + + + yarn.nodemanager.vmem-check-enabled + false + + + yarn.log-aggregation-enable + true + + + yarn.timeline-service.enabled + true + + + yarn.timeline-service.hostname + ranger-hadoop + + + yarn.timeline-service.http-cross-origin.enabled + true + + + yarn.resourcemanager.system-metrics-publisher.enabled + true + EOF mkdir -p /opt/hadoop/logs chown -R hdfs:hadoop /opt/hadoop/ chmod g+w /opt/hadoop/logs +# user logs directory permissions for NodeManager health +mkdir -p ${HADOOP_HOME}/logs/userlogs +chown -R yarn:hadoop ${HADOOP_HOME}/logs/userlogs +chmod -R 777 ${HADOOP_HOME}/logs/userlogs + +# Install Tez JARs for YARN NodeManager +echo "Installing Tez JARs for YARN NodeManager..." +if [ -d "/opt/tez" ]; then + echo "Copying Tez JARs to YARN lib directory..." + cp /opt/tez/lib/*.jar /opt/hadoop/share/hadoop/yarn/lib/ 2>/dev/null || echo 'Some Tez lib JARs may already exist' + cp /opt/tez/*.jar /opt/hadoop/share/hadoop/yarn/lib/ 2>/dev/null || echo 'Some Tez JARs may already exist' + + # Set up Tez environment + export TEZ_HOME=/opt/tez + export TEZ_CONF_DIR=${TEZ_HOME}/conf + mkdir -p ${TEZ_CONF_DIR} + + echo "Tez JARs installed successfully for YARN NodeManager" +else + echo "WARNING: Tez directory not found at /opt/tez" +fi cd ${RANGER_HOME}/ranger-hdfs-plugin ./enable-hdfs-plugin.sh diff --git a/dev-support/ranger-docker/scripts/ranger-hive-setup.sh b/dev-support/ranger-docker/scripts/ranger-hive-setup.sh index c0e7ee4063..cb4d7e8f92 100755 --- a/dev-support/ranger-docker/scripts/ranger-hive-setup.sh +++ b/dev-support/ranger-docker/scripts/ranger-hive-setup.sh @@ -35,6 +35,134 @@ EOF cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hive-site.xml cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hiveserver2-site.xml + +# Configure Tez +mkdir -p ${TEZ_HOME}/conf + +# Create Tez configuration directory for Hadoop +mkdir -p ${HADOOP_HOME}/etc/hadoop + +# Create mapred-site.xml for YARN integration +cat < ${HADOOP_HOME}/etc/hadoop/mapred-site.xml + + + mapreduce.framework.name + yarn + + + mapreduce.application.classpath + \$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:\$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/* + + + yarn.app.mapreduce.am.env + HADOOP_MAPRED_HOME=/opt/hadoop + + + mapreduce.map.env + HADOOP_MAPRED_HOME=/opt/hadoop + + + mapreduce.reduce.env + HADOOP_MAPRED_HOME=/opt/hadoop + + +EOF + +# Create yarn-site.xml for YARN ResourceManager connection +cat < ${HADOOP_HOME}/etc/hadoop/yarn-site.xml + + + yarn.resourcemanager.hostname + ranger-hadoop + + + yarn.resourcemanager.address + ranger-hadoop:8032 + + +EOF + +# Fix tez-site.xml to use absolute HDFS path (critical for Tez to find libraries) +cat < ${TEZ_HOME}/conf/tez-site.xml + + + + tez.lib.uris + hdfs://ranger-hadoop:9000/apps/tez/apache-tez-${TEZ_VERSION}-bin.tar.gz + Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. + + + tez.use.cluster.hadoop-libs + true + Use Hadoop libraries provided by cluster instead of those packaged with Tez + + + tez.am.resource.memory.mb + 1024 + The amount of memory to be used by the AppMaster + + + tez.am.java.opts + -Xmx768m + Java opts for the Tez AppMaster process + + + tez.task.resource.memory.mb + 1024 + The amount of memory to be used by tasks + + + tez.task.launch.cmd-opts + -Xmx768m + Java opts for tasks + + + tez.staging-dir + /tmp/hive + The staging directory for Tez applications in HDFS. + + +EOF + +# Copy Tez JARs to Hive lib directory +cp ${TEZ_HOME}/lib/tez-*.jar ${HIVE_HOME}/lib/ +cp ${TEZ_HOME}/tez-*.jar ${HIVE_HOME}/lib/ + +# Copy all Hadoop configurations to Hive conf directory so Hive can find them +cp ${HADOOP_HOME}/etc/hadoop/core-site.xml ${HIVE_HOME}/conf/ +cp ${HADOOP_HOME}/etc/hadoop/mapred-site.xml ${HIVE_HOME}/conf/ +cp ${HADOOP_HOME}/etc/hadoop/yarn-site.xml ${HIVE_HOME}/conf/ +cp ${TEZ_HOME}/conf/tez-site.xml ${HIVE_HOME}/conf/ + +# Upload Tez libraries to HDFS +su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /apps/tez" hdfs + +# Recreate Tez tarball if it doesn't exist (it gets removed during Docker build) +if [ ! -f "/opt/apache-tez-${TEZ_VERSION}-bin.tar.gz" ]; then + echo "Recreating Tez tarball for HDFS upload..." + cd /opt + tar czf apache-tez-${TEZ_VERSION}-bin.tar.gz apache-tez-${TEZ_VERSION}-bin/ +fi + +su -c "${HADOOP_HOME}/bin/hdfs dfs -put /opt/apache-tez-${TEZ_VERSION}-bin.tar.gz /apps/tez/" hdfs +su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod -R 755 /apps/tez" hdfs + +# Create HDFS user directory for hive +su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /user/hive" hdfs +su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod -R 777 /user/hive" hdfs + +# Create HDFS /tmp/hive directory for Tez staging +su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /tmp/hive" hdfs +su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod -R 777 /tmp/hive" hdfs + +# Fix /tmp directory permissions for Ranger (critical for INSERT operations) +su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod 777 /tmp" hdfs + +# Create /user/root directory for YARN job execution +su -c "${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /user/root" hdfs +su -c "${HADOOP_HOME}/bin/hdfs dfs -chmod 777 /user/root" hdfs + +# Initialize Hive schema su -c "${HIVE_HOME}/bin/schematool -dbType ${RANGER_DB_TYPE} -initSchema" hive mkdir -p /opt/hive/logs diff --git a/dev-support/ranger-docker/scripts/tez-site.xml b/dev-support/ranger-docker/scripts/tez-site.xml new file mode 100644 index 0000000000..80faf26e80 --- /dev/null +++ b/dev-support/ranger-docker/scripts/tez-site.xml @@ -0,0 +1,93 @@ + + + + + tez.lib.uris + ${fs.defaultFS}/apps/tez/apache-tez-${TEZ_VERSION}-bin.tar.gz + Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. + + + + tez.use.cluster.hadoop-libs + true + Use Hadoop libraries provided by cluster instead of those packaged with Tez + + + + tez.am.resource.memory.mb + 1024 + The amount of memory to be used by the AppMaster + + + + tez.am.java.opts + -Xmx768m + Java opts for the Tez AppMaster process + + + + tez.task.resource.memory.mb + 1024 + The amount of memory to be used by tasks + + + + tez.task.launch.cmd-opts + -Xmx768m + Java opts for tasks + + + + tez.runtime.io.sort.mb + 256 + The size of the sort buffer when output needs to be sorted + + + + tez.runtime.unordered.output.buffer.size-mb + 100 + The size of the buffer when output does not require to be sorted + + + + tez.session.am.dag.submit.timeout.secs + 300 + Time to wait (in seconds) for AM to submit a DAG before timing out + + + + tez.am.container.reuse.enabled + true + Whether to reuse containers for tasks belonging to the same session + + + + tez.am.container.reuse.rack-fallback.enabled + true + Whether to fallback to rack local containers when node local containers are not available + + + + tez.am.container.reuse.non-local-fallback.enabled + false + Whether to fallback to non-local containers when rack local containers are not available + + From 83d17bb236abd4071d74d6c99430e4042e936621 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Sun, 7 Sep 2025 17:52:13 -0700 Subject: [PATCH 2/5] RANGER-5310:Include Apache Tez as the process framework for ranger-hive docker - Review comments adddress, hadoop and hive ssh issue while startup addressed, removed not need configs --- dev-support/ranger-docker/.env | 1 + .../ranger-docker/Dockerfile.ranger-hive | 1 - .../ranger-docker/scripts/create-users.sh | 35 ++--- .../scripts/hive-site-metastore-mysql.xml | 132 ------------------ .../scripts/ranger-hadoop-setup.sh | 4 +- .../ranger-docker/scripts/ranger-hadoop.sh | 34 ++++- .../ranger-docker/scripts/ranger-hive.sh | 80 ++++++++++- 7 files changed, 122 insertions(+), 165 deletions(-) mode change 100755 => 100644 dev-support/ranger-docker/scripts/create-users.sh delete mode 100644 dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index e65b83627b..85e3008b5a 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -25,6 +25,7 @@ SOLR_VERSION=8.11.3 HADOOP_VERSION=3.3.6 HBASE_VERSION=2.6.0 HIVE_VERSION=4.0.1 +KAFKA_VERSION=2.8.2 HIVE_HADOOP_VERSION=3.3.6 TEZ_VERSION=0.10.4 KNOX_VERSION=2.0.0 diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive index c52e252ff3..d3ba8b458d 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hive +++ b/dev-support/ranger-docker/Dockerfile.ranger-hive @@ -39,7 +39,6 @@ COPY ./scripts/ranger-hive-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-hive.sh /home/ranger/scripts/ COPY ./scripts/ranger-hive-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/hive-site-${RANGER_DB_TYPE}.xml /home/ranger/scripts/hive-site.xml -COPY ./scripts/hive-site-metastore-${RANGER_DB_TYPE}.xml /home/ranger/scripts/hive-site-metastore.xml COPY ./scripts/create-users.sh /home/ranger/scripts/ RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ diff --git a/dev-support/ranger-docker/scripts/create-users.sh b/dev-support/ranger-docker/scripts/create-users.sh old mode 100755 new mode 100644 index dddca21e24..8f11b8b5e8 --- a/dev-support/ranger-docker/scripts/create-users.sh +++ b/dev-support/ranger-docker/scripts/create-users.sh @@ -22,41 +22,22 @@ # Function to create a user for testing. create_user_if_not_exists() { local username=$1 - local uid=$2 - local gid=$3 - local home_dir=$4 + local home_dir=$2 if ! id "$username" &>/dev/null; then - echo "Creating user: $username (uid:$uid, gid:$gid)" - useradd -u "$uid" -g "$gid" -m -d "$home_dir" -s /bin/bash "$username" + echo "Creating user: $username" + useradd -m -d "$home_dir" -s /bin/bash "$username" - # Set a default password (same as username for demo purposes) + # Set a default password echo "$username:$username" | chpasswd - # Add user to hadoop group for HDFS access - if getent group hadoop &>/dev/null; then - usermod -a -G hadoop "$username" - fi - - # Create .ssh directory and set proper permissions - mkdir -p "$home_dir/.ssh" - chmod 700 "$home_dir/.ssh" - chown "$username:$gid" "$home_dir/.ssh" - echo "User $username created successfully" else echo "User $username already exists" fi } -# Ensure hadoop group exists (gid 1001 is used by hdfs, yarn, hive users) -if ! getent group hadoop &>/dev/null; then - groupadd -g 1001 hadoop - echo "Created hadoop group" -fi - -# Create alice user (uid: 2001, gid: 1001 - hadoop group) -create_user_if_not_exists "alice" 2001 1001 "/home/alice" - -# Create abram user (uid: 2002, gid: 1001 - hadoop group) -create_user_if_not_exists "abram" 2002 1001 "/home/abram" +# Create alice user +create_user_if_not_exists "alice" "/home/alice" +# Create abram user +create_user_if_not_exists "abram" "/home/abram" diff --git a/dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml b/dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml deleted file mode 100644 index 1049239f58..0000000000 --- a/dev-support/ranger-docker/scripts/hive-site-metastore-mysql.xml +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - javax.jdo.option.ConnectionURL - jdbc:mysql://ranger-db/hive - - - javax.jdo.option.ConnectionDriverName - com.mysql.jdbc.Driver - - - javax.jdo.option.ConnectionUserName - hive - - - javax.jdo.option.ConnectionPassword - rangerR0cks! - - - - - hive.server2.enable.doAs - false - - - - - hive.metastore.warehouse.dir - file:///warehouse/tablespace/managed/hive - - - hive.metastore.warehouse.external.dir - file:///warehouse/tablespace/external/hive - - - - hive.zookeeper.quorum - ranger-zk.example.com - - - hive.zookeeper.client.port - 2181 - - - - - hive.execution.engine - tez - Execution engine to use for Hive queries - - - - hive.tez.container.size - 1024 - By default Tez will spawn containers of the size of a mapper - - - - hive.tez.java.opts - -Xmx768m - Java command line options for Tez - - - - hive.tez.log.level - INFO - Log level for Tez - - - - hive.prewarm.enabled - false - Enables container prewarm for Tez - - - - hive.prewarm.numcontainers - 3 - Controls the number of containers to prewarm for Tez - - - - hive.tez.auto.reducer.parallelism - true - Turn on Tez' auto reducer parallelism feature - - - - hive.tez.min.reducer.per.query - 2 - Controls the minimum number of reducers for Tez - - - - hive.tez.max.reducer.per.query - 999 - Controls the maximum number of reducers for Tez - - - - - hive.tez.exec.print.summary - true - Whether to print a summary of the plan execution - - - - tez.queue.name - default - YARN queue name for Tez jobs - - diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh index d87548b80f..e08b4ac859 100755 --- a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh +++ b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh @@ -115,8 +115,8 @@ chmod -R 777 ${HADOOP_HOME}/logs/userlogs echo "Installing Tez JARs for YARN NodeManager..." if [ -d "/opt/tez" ]; then echo "Copying Tez JARs to YARN lib directory..." - cp /opt/tez/lib/*.jar /opt/hadoop/share/hadoop/yarn/lib/ 2>/dev/null || echo 'Some Tez lib JARs may already exist' - cp /opt/tez/*.jar /opt/hadoop/share/hadoop/yarn/lib/ 2>/dev/null || echo 'Some Tez JARs may already exist' + cp /opt/tez/lib/*.jar /opt/hadoop/share/hadoop/yarn/lib/ 2>/dev/null + cp /opt/tez/*.jar /opt/hadoop/share/hadoop/yarn/lib/ 2>/dev/null # Set up Tez environment export TEZ_HOME=/opt/tez diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh index 164c25addb..043d16aea2 100755 --- a/dev-support/ranger-docker/scripts/ranger-hadoop.sh +++ b/dev-support/ranger-docker/scripts/ranger-hadoop.sh @@ -18,6 +18,39 @@ CREATE_HDFS_DIR=false +# Always ensure SSH daemon is running (required for Hadoop services) +echo "Starting SSH daemon..." +# Create SSH privilege separation directory if it doesn't exist +mkdir -p /run/sshd +/usr/sbin/sshd + +if [ -f /home/hdfs/.ssh/id_rsa ]; then + echo "Waiting for SSH daemon to be ready..." + SSH_READY=false + for i in {1..30}; do + if su -c "ssh -o ConnectTimeout=2 -o StrictHostKeyChecking=no localhost exit" hdfs 2>/dev/null; then + echo "SSH daemon is ready for hdfs service..." + SSH_READY=true + break + fi + echo "Waiting for SSH daemon... ($i/30)" + sleep 2 + done + + if [ "$SSH_READY" = false ]; then + echo "WARNING: SSH daemon did not become ready within 60 seconds, Hadoop Services may fail to start properly...." + echo "Attempting to restart SSH daemon..." + pkill sshd 2>/dev/null || true + # Ensure SSH privilege separation directory exists + mkdir -p /run/sshd + /usr/sbin/sshd + sleep 3 + fi +else + echo "SSH keys not yet generated, skipping SSH connectivity test" + sleep 2 +fi + if [ ! -e ${HADOOP_HOME}/.setupDone ] then su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hdfs @@ -29,7 +62,6 @@ then su -c "chmod 0600 ~/.ssh/authorized_keys" yarn ssh-keygen -A - /usr/sbin/sshd -D & # pdsh is unavailable with microdnf in rhel based image. echo "ssh" > /etc/pdsh/rcmd_default diff --git a/dev-support/ranger-docker/scripts/ranger-hive.sh b/dev-support/ranger-docker/scripts/ranger-hive.sh index 6e8dc4f847..249403faed 100755 --- a/dev-support/ranger-docker/scripts/ranger-hive.sh +++ b/dev-support/ranger-docker/scripts/ranger-hive.sh @@ -17,7 +17,45 @@ # limitations under the License. if [ "${OS_NAME}" = "UBUNTU" ]; then + echo "Starting SSH service (Ubuntu)..." service ssh start +else + echo "Starting SSH daemon (RHEL/CentOS)..." + # Create SSH privilege separation directory if it doesn't exist + mkdir -p /run/sshd + /usr/sbin/sshd +fi + +# Wait for SSH daemon to be fully ready before proceeding +if [ -f /home/hdfs/.ssh/id_rsa ]; then + echo "Waiting for SSH daemon to be ready..." + SSH_READY=false + for i in {1..30}; do + if su -c "ssh -o ConnectTimeout=2 -o StrictHostKeyChecking=no localhost exit" hdfs 2>/dev/null; then + echo "SSH daemon is ready for hdfs service..." + SSH_READY=true + break + fi + echo "Waiting for SSH daemon... ($i/30)" + sleep 2 + done + + if [ "$SSH_READY" = false ]; then + echo "WARNING: SSH daemon did not become ready within 60 seconds, Hive Services may fail to start properly...." + echo "Attempting to restart SSH daemon..." + pkill sshd 2>/dev/null || true + if [ "${OS_NAME}" = "UBUNTU" ]; then + service ssh start + else + # Ensure SSH privilege separation directory exists + mkdir -p /run/sshd + /usr/sbin/sshd + fi + sleep 3 + fi +else + echo "SSH keys not yet generated, skipping SSH connectivity test" + sleep 2 fi if [ ! -e ${HIVE_HOME}/.setupDone ] @@ -28,13 +66,16 @@ then if [ "${OS_NAME}" = "RHEL" ]; then ssh-keygen -A - /usr/sbin/sshd fi su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" yarn su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" yarn su -c "chmod 0600 ~/.ssh/authorized_keys" yarn + su -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa" hive + su -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys" hive + su -c "chmod 0600 ~/.ssh/authorized_keys" hive + # pdsh is unavailable with microdnf in rhel based image. echo "ssh" > /etc/pdsh/rcmd_default @@ -50,14 +91,49 @@ fi cd "${HIVE_HOME}" || exit # Start Hive MetaStore +echo "Starting Hive MetaStore..." su -c "nohup ${HIVE_HOME}/bin/hive --service metastore > metastore.log 2>&1 &" hive # Start HiveServer2 +echo "Starting HiveServer2..." su -c "nohup ${HIVE_HOME}/bin/hiveserver2 > hive-server2.log 2>&1 &" hive +# Wait for services to initialize +echo "Waiting for Hive services to initialize..." sleep 10 -HIVE_SERVER2_PID=`ps -ef | grep -v grep | grep -i "org.apache.hive.service.server.HiveServer2" | awk '{print $2}'` +# Verify Hive services are running and ready +echo "Verifying Hive services are ready for beeline connections..." +METASTORE_PID=`ps -ef | grep -v grep | grep -i "org.apache.hadoop.hive.metastore.HiveMetaStore" | awk '{print $2}'` +HIVE_SERVER2_PID=`ps -ef | grep -v grep | grep -i "org.apache.hive.service.server.HiveServer2" | awk '{print $2}'` + +if [ -n "$METASTORE_PID" ]; then + echo "Hive MetaStore is running (PID: $METASTORE_PID)" +else + echo "WARNING: Hive MetaStore process not found!" +fi + +if [ -n "$HIVE_SERVER2_PID" ]; then + echo "HiveServer2 is running (PID: $HIVE_SERVER2_PID)" +else + echo "WARNING: HiveServer2 process not found!" +fi + +# Additional verification: Check if HiveServer2 is listening on port 10000 +echo "Checking if HiveServer2 is listening on port 10000..." +for i in {1..30}; do + if timeout 2 bash -c "echo > /dev/tcp/localhost/10000" 2>/dev/null; then + echo "HiveServer2 is ready and listening on port 10000...." + break + fi + if [ $i -eq 30 ]; then + echo "WARNING: HiveServer2 is not listening on port 10000 after 60 seconds" + echo "Beeline connections may fail. Check metastore.log and hive-server2.log for errors." + else + echo "Waiting for HiveServer2 to listen on port 10000... ($i/30)" + sleep 2 + fi +done # prevent the container from exiting if [ -z "$HIVE_SERVER2_PID" ] From 482d27f08eb05ecd1cbffc0b7752dec06840ea81 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Tue, 30 Sep 2025 22:42:52 -0700 Subject: [PATCH 3/5] RANGER-5310:Include Apache Tez as the process framework for ranger-hive docker - changes to use ranger base image for user creation, fix issue with usage of ranger base image in other containers --- dev-support/ranger-docker/.env | 77 ++++++++++++------- dev-support/ranger-docker/Dockerfile.ranger | 7 ++ .../ranger-docker/Dockerfile.ranger-hadoop | 10 +-- .../ranger-docker/Dockerfile.ranger-hbase | 15 ++-- .../ranger-docker/Dockerfile.ranger-hive | 16 ++-- .../ranger-docker/Dockerfile.ranger-kafka | 15 ++-- .../ranger-docker/Dockerfile.ranger-kms | 15 ++-- .../ranger-docker/Dockerfile.ranger-knox | 15 ++-- .../ranger-docker/Dockerfile.ranger-tagsync | 12 ++- .../ranger-docker/Dockerfile.ranger-usersync | 19 ++--- dev-support/ranger-docker/config/my.cnf | 2 + .../docker-compose.ranger-db.yml | 2 +- .../docker-compose.ranger-hadoop.yml | 2 - .../docker-compose.ranger-hive.yml | 2 - 14 files changed, 114 insertions(+), 95 deletions(-) diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index 85e3008b5a..dd985f3f76 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -11,47 +11,66 @@ RANGER_BASE_IMAGE=apache/ranger-base RANGER_BASE_VERSION=20250707-1-8 # Java version used to build Apache Ranger is present as suffix: -8, valid values for suffix: -8, -11, -17 RANGER_BASE_BUILD_VERSION=20250707-1-8 +RANGER_VERSION=3.0.0-SNAPSHOT -# third party image versions -MARIADB_VERSION=10.7.3 -POSTGRES_VERSION=12 -ORACLE_VERSION=23.6 -SQLSERVER_VERSION=2019-latest -ENABLE_DB_MOUNT=true -ZK_VERSION=3.9.2 -SOLR_VERSION=8.11.3 - -# service versions +# Hadoop Configuration HADOOP_VERSION=3.3.6 -HBASE_VERSION=2.6.0 -HIVE_VERSION=4.0.1 -KAFKA_VERSION=2.8.2 +HDFS_PLUGIN_VERSION=3.0.0-SNAPSHOT +YARN_PLUGIN_VERSION=3.0.0-SNAPSHOT + +# Hive Configuration HIVE_HADOOP_VERSION=3.3.6 +HIVE_VERSION=4.0.1 +HIVE_PLUGIN_VERSION=3.0.0-SNAPSHOT + +# Tez Configuration TEZ_VERSION=0.10.4 + +# HBase Configuration +HBASE_VERSION=2.6.0 +HBASE_PLUGIN_VERSION=3.0.0-SNAPSHOT + +# Kafka Configuration +KAFKA_VERSION=2.8.2 +KAFKA_PLUGIN_VERSION=3.0.0-SNAPSHOT + +# Knox Configuration KNOX_VERSION=2.0.0 -TRINO_VERSION=377 -OZONE_VERSION=1.4.0 -OZONE_RUNNER_VERSION=20230615-1 -OZONE_RUNNER_IMAGE=apache/ozone-runner -OZONE_OPTS= +KNOX_PLUGIN_VERSION=3.0.0-SNAPSHOT -# versions of ranger services -RANGER_VERSION=3.0.0-SNAPSHOT +# KMS Configuration KMS_VERSION=3.0.0-SNAPSHOT + +# Usersync Configuration USERSYNC_VERSION=3.0.0-SNAPSHOT + +# Tagsync Configuration TAGSYNC_VERSION=3.0.0-SNAPSHOT -# plugin versions -HDFS_PLUGIN_VERSION=3.0.0-SNAPSHOT -YARN_PLUGIN_VERSION=3.0.0-SNAPSHOT -HIVE_PLUGIN_VERSION=3.0.0-SNAPSHOT -HBASE_PLUGIN_VERSION=3.0.0-SNAPSHOT -KAFKA_PLUGIN_VERSION=3.0.0-SNAPSHOT -KNOX_PLUGIN_VERSION=3.0.0-SNAPSHOT -TRINO_PLUGIN_VERSION=3.0.0-SNAPSHOT +# Solr Configuration +SOLR_VERSION=8.11.2 + +# Zookeeper Configuration +ZK_VERSION=3.8.4 + +# Database Versions +POSTGRES_VERSION=13.16 +MARIADB_VERSION=10.6 +ORACLE_VERSION=21.3.0-xe +SQLSERVER_VERSION=2022-latest + +# Ozone Configuration +OZONE_VERSION=1.4.0 OZONE_PLUGIN_VERSION=3.0.0-SNAPSHOT +OZONE_RUNNER_IMAGE=apache/ozone-runner +OZONE_RUNNER_VERSION=20240625 + +# Trino Configuration +TRINO_VERSION=435 +TRINO_PLUGIN_VERSION=3.0.0-SNAPSHOT -# To enable debug logs +# Debug Configuration DEBUG_ADMIN=false DEBUG_USERSYNC=false DEBUG_TAGSYNC=false +ENABLE_FILE_SYNC_SOURCE=false diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index fecc7b2655..d3b0bd2a87 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -23,6 +23,10 @@ ARG RANGER_VERSION ARG RANGER_DB_TYPE ARG TARGETARCH +# Set Ranger environment variables +ENV RANGER_HOME=/opt/ranger +ENV RANGER_SCRIPTS=/home/ranger/scripts + COPY ./dist/version /home/ranger/dist/ COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/ @@ -31,6 +35,9 @@ COPY ./scripts/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPT COPY ./scripts/create-ranger-services.py ${RANGER_SCRIPTS}/ COPY ./scripts/create-users.sh ${RANGER_SCRIPTS}/ +# Switch to root to create system directories +USER root + RUN tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} \ && ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin \ && rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop index 3a5761f03c..ed8aa622aa 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop +++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} +FROM ranger ARG HADOOP_VERSION ARG HDFS_PLUGIN_VERSION @@ -35,7 +33,9 @@ COPY ./scripts/ranger-hadoop.sh /home/ranger/scripts/ COPY ./scripts/ranger-hadoop-mkdir.sh /home/ranger/scripts/ COPY ./scripts/ranger-hdfs-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/ranger-yarn-plugin-install.properties /home/ranger/scripts/ -COPY ./scripts/create-users.sh /home/ranger/scripts/ + +# Switch to root to create system directories +USER root RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \ ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \ @@ -51,8 +51,6 @@ RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ rm -f /home/ranger/dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-yarn-plugin-install.properties /opt/ranger/ranger-yarn-plugin/install.properties && \ chmod 744 ${RANGER_SCRIPTS}/ranger-hadoop-setup.sh ${RANGER_SCRIPTS}/ranger-hadoop.sh ${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh && \ - chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ - ${RANGER_SCRIPTS}/create-users.sh && \ chown hdfs:hadoop ${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh RUN apt-get update && \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase index 0d1884f0ab..9b105e3aff 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hbase +++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} +FROM ranger ARG HBASE_VERSION ARG HBASE_PLUGIN_VERSION @@ -29,9 +27,11 @@ COPY ./downloads/hbase-${HBASE_VERSION}-bin.tar.gz /home/ranger/dis COPY ./scripts/ranger-hbase-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/ COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/ -COPY ./scripts/create-users.sh /home/ranger/scripts/ COPY ./scripts/hbase-site.xml /home/ranger/scripts/ +# Switch to root to create system directories +USER root + RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \ ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \ rm -f /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz && \ @@ -39,9 +39,7 @@ RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/op ln -s /opt/ranger/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin /opt/ranger/ranger-hbase-plugin && \ rm -f /home/ranger/dist/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-hbase-plugin-install.properties /opt/ranger/ranger-hbase-plugin/install.properties && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-hbase-setup.sh ${RANGER_SCRIPTS}/ranger-hbase.sh && \ - chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ - ${RANGER_SCRIPTS}/create-users.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-hbase-setup.sh ${RANGER_SCRIPTS}/ranger-hbase.sh RUN apt-get update && \ apt-get install -y --no-install-recommends openssh-server && \ @@ -51,4 +49,7 @@ RUN apt-get update && \ ENV HBASE_HOME=/opt/hbase ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hbase/bin +# Ensure container runs as root for SSH and service management +USER root + ENTRYPOINT [ "/home/ranger/scripts/ranger-hbase.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive index d3ba8b458d..aefef99957 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hive +++ b/dev-support/ranger-docker/Dockerfile.ranger-hive @@ -13,11 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_DB_TYPE +# Only the ARGs needed for this build stage -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} +FROM ranger ARG HIVE_VERSION ARG HIVE_HADOOP_VERSION @@ -39,7 +37,9 @@ COPY ./scripts/ranger-hive-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-hive.sh /home/ranger/scripts/ COPY ./scripts/ranger-hive-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/hive-site-${RANGER_DB_TYPE}.xml /home/ranger/scripts/hive-site.xml -COPY ./scripts/create-users.sh /home/ranger/scripts/ + +# Switch to root to create system directories +USER root RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \ @@ -57,9 +57,7 @@ RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz ln -s /opt/ranger/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \ rm -f /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-hive-plugin-install.properties /opt/ranger/ranger-hive-plugin/install.properties && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-hive-setup.sh ${RANGER_SCRIPTS}/ranger-hive.sh && \ - chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ - ${RANGER_SCRIPTS}/create-users.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-hive-setup.sh ${RANGER_SCRIPTS}/ranger-hive.sh ENV HIVE_HOME=/opt/hive ENV HADOOP_HOME=/opt/hadoop @@ -67,5 +65,7 @@ ENV TEZ_HOME=/opt/tez ENV TEZ_CONF_DIR=/opt/tez/conf ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin:/opt/tez/bin +# Ensure container runs as root for SSH and service management +USER root ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka index fdc8258e56..ba38b87244 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kafka +++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} +FROM ranger ARG KAFKA_VERSION ARG KAFKA_PLUGIN_VERSION @@ -29,7 +27,9 @@ COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/ranger/dist COPY ./scripts/ranger-kafka-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-kafka.sh /home/ranger/scripts/ COPY ./scripts/ranger-kafka-plugin-install.properties /home/ranger/scripts/ -COPY ./scripts/create-users.sh /home/ranger/scripts/ + +# Switch to root to create system directories +USER root RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \ ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \ @@ -38,12 +38,13 @@ RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ ln -s /opt/ranger/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin /opt/ranger/ranger-kafka-plugin && \ rm -f /home/ranger/dist/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-kafka-plugin-install.properties /opt/ranger/ranger-kafka-plugin/install.properties && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-kafka-setup.sh ${RANGER_SCRIPTS}/ranger-kafka.sh && \ - chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ - ${RANGER_SCRIPTS}/create-users.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-kafka-setup.sh ${RANGER_SCRIPTS}/ranger-kafka.sh ENV KAFKA_HOME=/opt/kafka ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin +# Ensure container runs as root for SSH and service management +USER root + ENTRYPOINT [ "/home/ranger/scripts/ranger-kafka.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms index e1d97c650b..93ebf270ad 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kms +++ b/dev-support/ranger-docker/Dockerfile.ranger-kms @@ -13,10 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_DB_TYPE -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} AS ranger-kms + +FROM ranger AS ranger-kms ARG KMS_VERSION ARG RANGER_DB_TYPE @@ -26,7 +24,9 @@ COPY ./dist/ranger-${KMS_VERSION}-kms.tar.gz /home/ranger/dist COPY ./scripts/ranger-kms.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-kms-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-kms-install.properties -COPY ./scripts/create-users.sh /home/ranger/scripts/ + +# Switch to root to create system directories +USER root RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${KMS_VERSION}-kms ${RANGER_HOME}/kms && \ @@ -39,9 +39,8 @@ RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RA ln -s /etc/init.d/ranger-kms /etc/rc3.d/S88ranger-kms && \ ln -s /etc/init.d/ranger-kms /etc/rc3.d/K90ranger-kms && \ ln -s ${RANGER_HOME}/kms/ranger-kms-services.sh /usr/bin/ranger-kms-services.sh && \ - chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-kms.sh && \ - ${RANGER_SCRIPTS}/create-users.sh + chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ /etc/ranger && \ + chmod 744 ${RANGER_SCRIPTS}/ranger-kms.sh FROM ranger-kms AS ranger_postgres COPY ./downloads/postgresql-42.2.16.jre7.jar /home/ranger/dist/ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-knox b/dev-support/ranger-docker/Dockerfile.ranger-knox index 13bbf006aa..c5bc34a970 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-knox +++ b/dev-support/ranger-docker/Dockerfile.ranger-knox @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} +FROM ranger ARG KNOX_VERSION ARG KNOX_PLUGIN_VERSION @@ -25,7 +23,6 @@ ARG KNOX_PLUGIN_VERSION COPY ./dist/version /home/ranger/dist/ COPY ./dist/ranger-${KNOX_PLUGIN_VERSION}-knox-plugin.tar.gz /home/ranger/dist/ COPY ./downloads/knox-${KNOX_VERSION}.tar.gz /home/ranger/dist/ -COPY ./scripts/create-users.sh /home/ranger/scripts/ COPY ./scripts/ranger-knox-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-knox.sh /home/ranger/scripts/ @@ -33,6 +30,9 @@ COPY ./scripts/ranger-knox-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/ranger-knox-expect.py /home/ranger/scripts/ COPY ./scripts/ranger-knox-sandbox.xml /home/ranger/scripts/ +# Switch to root to create system directories +USER root + RUN tar xvfz /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz --directory=/opt/ && \ ln -s /opt/knox-${KNOX_VERSION} /opt/knox && \ rm -f /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz && \ @@ -41,9 +41,7 @@ RUN tar xvfz /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz --directory=/opt/ && rm -f /home/ranger/dist/ranger-${KNOX_PLUGIN_VERSION}-knox-plugin.tar.gz && \ cp -f /home/ranger/scripts/ranger-knox-plugin-install.properties /opt/ranger/ranger-knox-plugin/install.properties && \ cp -f /home/ranger/scripts/ranger-knox-sandbox.xml /opt/knox/conf/topologies/sandbox.xml && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-knox-setup.sh ${RANGER_SCRIPTS}/ranger-knox.sh ${RANGER_SCRIPTS}/ranger-knox-expect.py && \ - chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ - ${RANGER_SCRIPTS}/create-users.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-knox-setup.sh ${RANGER_SCRIPTS}/ranger-knox.sh ${RANGER_SCRIPTS}/ranger-knox-expect.py ENV KNOX_HOME=/opt/knox ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/knox/bin @@ -53,4 +51,7 @@ RUN chmod a+rwx /home/ranger/scripts/ranger-knox-expect.py RUN pip3 install pexpect RUN python3 /home/ranger/scripts/ranger-knox-expect.py +# Ensure container runs as root for SSH and service management +USER root + ENTRYPOINT [ "/home/ranger/scripts/ranger-knox.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-tagsync b/dev-support/ranger-docker/Dockerfile.ranger-tagsync index 5b0573252b..6101fba6dd 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-tagsync +++ b/dev-support/ranger-docker/Dockerfile.ranger-tagsync @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} +FROM ranger ARG TAGSYNC_VERSION @@ -26,7 +24,9 @@ COPY ./dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz /home/ranger/dist/ COPY ./scripts/ranger-tagsync.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-tagsync-install.properties ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-tagsync-tags.json ${RANGER_SCRIPTS}/ -COPY ./scripts/create-users.sh ${RANGER_SCRIPTS}/ + +# Switch to root to create system directories +USER root RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${TAGSYNC_VERSION}-tagsync ${RANGER_HOME}/tagsync && \ @@ -44,9 +44,7 @@ RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --direct ln -s /etc/init.d/ranger-tagsync /etc/rc3.d/K00ranger-tagsync && \ ln -s ${RANGER_HOME}/tagsync/ranger-tagsync-services.sh /usr/bin/ranger-tagsync-services.sh && \ chown -R ranger:ranger ${RANGER_HOME}/tagsync/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ /etc/ranger /etc/init.d/ranger-tagsync && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-tagsync.sh && \ - chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ - ${RANGER_SCRIPTS}/create-users.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-tagsync.sh USER ranger diff --git a/dev-support/ranger-docker/Dockerfile.ranger-usersync b/dev-support/ranger-docker/Dockerfile.ranger-usersync index 893d440578..37b258d3e6 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-usersync +++ b/dev-support/ranger-docker/Dockerfile.ranger-usersync @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG RANGER_BASE_IMAGE -ARG RANGER_BASE_VERSION -FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} +FROM ranger ARG USERSYNC_VERSION @@ -26,16 +24,17 @@ COPY ./dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz /home/ranger/dist/ COPY ./scripts/ranger-usersync.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-usersync-install.properties ${RANGER_SCRIPTS}/ COPY ./scripts/ugsync-file-source.csv ${RANGER_SCRIPTS}/ -COPY ./scripts/create-users.sh ${RANGER_SCRIPTS}/ + +# Switch to root to create system directories +USER root RUN tar xvfz /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${USERSYNC_VERSION}-usersync ${RANGER_HOME}/usersync && \ rm -f /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz && \ cp -f ${RANGER_SCRIPTS}/ranger-usersync-install.properties ${RANGER_HOME}/usersync/install.properties && \ - mkdir -p /var/run/ranger /var/log/ranger/usersync /etc/ranger && \ - mkdir /etc/init.d || true && \ - mkdir /etc/rc2.d || true && \ - mkdir /etc/rc3.d || true && \ + mkdir -p /var/run/ranger /var/log/ranger/usersync && \ + mkdir -p /etc/ranger && \ + mkdir -p /etc/init.d /etc/rc2.d /etc/rc3.d && \ touch /etc/init.d/ranger-usersync && \ ln -s /etc/init.d/ranger-usersync /etc/rc2.d/S99ranger-usersync && \ ln -s /etc/init.d/ranger-usersync /etc/rc2.d/K00ranger-usersync && \ @@ -43,9 +42,7 @@ RUN tar xvfz /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz --dire ln -s /etc/init.d/ranger-usersync /etc/rc3.d/K00ranger-usersync && \ ln -s ${RANGER_HOME}/usersync/ranger-usersync-services.sh /usr/bin/ranger-usersync && \ chown -R ranger:ranger ${RANGER_HOME}/usersync/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ /etc/ranger /etc/init.d/ranger-usersync && \ - chmod 744 ${RANGER_SCRIPTS}/ranger-usersync.sh && \ - chmod +x ${RANGER_SCRIPTS}/create-users.sh && \ - ${RANGER_SCRIPTS}/create-users.sh + chmod 744 ${RANGER_SCRIPTS}/ranger-usersync.sh USER ranger diff --git a/dev-support/ranger-docker/config/my.cnf b/dev-support/ranger-docker/config/my.cnf index 77e958eaca..bd7d646d5c 100644 --- a/dev-support/ranger-docker/config/my.cnf +++ b/dev-support/ranger-docker/config/my.cnf @@ -19,5 +19,7 @@ collation-server = utf8_unicode_ci init-connect = 'SET NAMES utf8' character_set_server = utf8 +# Enable hostname resolution for user authentication (fixes ranger-db user issue) +skip-name-resolve = 0 # transaction-isolation = REPEATABLE-READ # transaction-isolation = READ-COMMITTED diff --git a/dev-support/ranger-docker/docker-compose.ranger-db.yml b/dev-support/ranger-docker/docker-compose.ranger-db.yml index 3688a2a6df..3a819afb18 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-db.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-db.yml @@ -25,7 +25,7 @@ services: args: - MARIADB_VERSION=${MARIADB_VERSION} image: ranger-mysql - command: --default-authentication-plugin=mysql_native_password + command: --default-authentication-plugin=mysql_native_password --skip-name-resolve=0 container_name: ranger-mysql hostname: ranger-db.example.com ports: diff --git a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml index 37fe6b5d83..1ab67d5b9e 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml @@ -4,8 +4,6 @@ services: context: . dockerfile: Dockerfile.ranger-hadoop args: - - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} - - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} - HADOOP_VERSION=${HADOOP_VERSION} - HDFS_PLUGIN_VERSION=${HDFS_PLUGIN_VERSION} - YARN_PLUGIN_VERSION=${YARN_PLUGIN_VERSION} diff --git a/dev-support/ranger-docker/docker-compose.ranger-hive.yml b/dev-support/ranger-docker/docker-compose.ranger-hive.yml index 6084630825..47e57a84dc 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-hive.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-hive.yml @@ -4,8 +4,6 @@ services: context: . dockerfile: Dockerfile.ranger-hive args: - - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} - - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} - HIVE_HADOOP_VERSION=${HIVE_HADOOP_VERSION} - HIVE_VERSION=${HIVE_VERSION} - HIVE_PLUGIN_VERSION=${HIVE_PLUGIN_VERSION} From cef975cb62ff885650e2aebc0c438691cdd2dd00 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Wed, 8 Oct 2025 14:23:32 -0700 Subject: [PATCH 4/5] RANGER-5310:Include Apache Tez as the process framework for ranger-hive docker - addressed review comment on issue related to base immage --- dev-support/ranger-docker/Dockerfile.ranger | 3 -- .../ranger-docker/Dockerfile.ranger-hadoop | 4 +- .../ranger-docker/Dockerfile.ranger-hbase | 4 +- .../ranger-docker/Dockerfile.ranger-hive | 4 +- .../ranger-docker/Dockerfile.ranger-kafka | 4 +- .../ranger-docker/Dockerfile.ranger-kms | 5 ++- .../ranger-docker/Dockerfile.ranger-knox | 4 +- .../ranger-docker/Dockerfile.ranger-tagsync | 4 +- .../ranger-docker/Dockerfile.ranger-usersync | 4 +- .../docker-compose.ranger-hadoop.yml | 2 + .../docker-compose.ranger-hive.yml | 2 + .../ranger-docker/scripts/create-users.sh | 43 ------------------- 12 files changed, 29 insertions(+), 54 deletions(-) delete mode 100644 dev-support/ranger-docker/scripts/create-users.sh diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index d3b0bd2a87..349cc7be09 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -33,7 +33,6 @@ COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/ COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-admin-install.properties COPY ./scripts/create-ranger-services.py ${RANGER_SCRIPTS}/ -COPY ./scripts/create-users.sh ${RANGER_SCRIPTS}/ # Switch to root to create system directories USER root @@ -45,8 +44,6 @@ RUN tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --direct && mkdir -p /var/run/ranger \ && mkdir -p /var/log/ranger \ && chown -R ranger:ranger ${RANGER_HOME}/admin/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ \ - && chmod +x ${RANGER_SCRIPTS}/create-users.sh \ - && ${RANGER_SCRIPTS}/create-users.sh \ && chmod 755 ${RANGER_SCRIPTS}/ranger.sh \ && mkdir -p /usr/share/java/ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop index ed8aa622aa..e036750d6c 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop +++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop @@ -14,7 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ranger +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG HADOOP_VERSION ARG HDFS_PLUGIN_VERSION diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase index 9b105e3aff..dccbc49442 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hbase +++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase @@ -14,7 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ranger +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG HBASE_VERSION ARG HBASE_PLUGIN_VERSION diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive index aefef99957..c562bafe42 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hive +++ b/dev-support/ranger-docker/Dockerfile.ranger-hive @@ -15,7 +15,9 @@ # limitations under the License. # Only the ARGs needed for this build stage -FROM ranger +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG HIVE_VERSION ARG HIVE_HADOOP_VERSION diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka index ba38b87244..6dc9f59c21 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kafka +++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka @@ -14,7 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ranger +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG KAFKA_VERSION ARG KAFKA_PLUGIN_VERSION diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms index 93ebf270ad..1758be9a32 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kms +++ b/dev-support/ranger-docker/Dockerfile.ranger-kms @@ -14,7 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ranger AS ranger-kms +ARG RANGER_DB_TYPE +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} AS ranger-kms ARG KMS_VERSION ARG RANGER_DB_TYPE diff --git a/dev-support/ranger-docker/Dockerfile.ranger-knox b/dev-support/ranger-docker/Dockerfile.ranger-knox index c5bc34a970..e3365939e5 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-knox +++ b/dev-support/ranger-docker/Dockerfile.ranger-knox @@ -14,7 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ranger +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG KNOX_VERSION ARG KNOX_PLUGIN_VERSION diff --git a/dev-support/ranger-docker/Dockerfile.ranger-tagsync b/dev-support/ranger-docker/Dockerfile.ranger-tagsync index 6101fba6dd..352322f925 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-tagsync +++ b/dev-support/ranger-docker/Dockerfile.ranger-tagsync @@ -14,7 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ranger +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG TAGSYNC_VERSION diff --git a/dev-support/ranger-docker/Dockerfile.ranger-usersync b/dev-support/ranger-docker/Dockerfile.ranger-usersync index 37b258d3e6..294c63c83d 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-usersync +++ b/dev-support/ranger-docker/Dockerfile.ranger-usersync @@ -14,7 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ranger +ARG RANGER_BASE_IMAGE +ARG RANGER_BASE_VERSION +FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} ARG USERSYNC_VERSION diff --git a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml index 1ab67d5b9e..37fe6b5d83 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml @@ -4,6 +4,8 @@ services: context: . dockerfile: Dockerfile.ranger-hadoop args: + - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} + - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} - HADOOP_VERSION=${HADOOP_VERSION} - HDFS_PLUGIN_VERSION=${HDFS_PLUGIN_VERSION} - YARN_PLUGIN_VERSION=${YARN_PLUGIN_VERSION} diff --git a/dev-support/ranger-docker/docker-compose.ranger-hive.yml b/dev-support/ranger-docker/docker-compose.ranger-hive.yml index 47e57a84dc..6084630825 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-hive.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-hive.yml @@ -4,6 +4,8 @@ services: context: . dockerfile: Dockerfile.ranger-hive args: + - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE} + - RANGER_BASE_VERSION=${RANGER_BASE_VERSION} - HIVE_HADOOP_VERSION=${HIVE_HADOOP_VERSION} - HIVE_VERSION=${HIVE_VERSION} - HIVE_PLUGIN_VERSION=${HIVE_PLUGIN_VERSION} diff --git a/dev-support/ranger-docker/scripts/create-users.sh b/dev-support/ranger-docker/scripts/create-users.sh deleted file mode 100644 index 8f11b8b5e8..0000000000 --- a/dev-support/ranger-docker/scripts/create-users.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script to create alice and abram users in ranger containers -# This script is designed to be run during container initialization - -# Function to create a user for testing. -create_user_if_not_exists() { - local username=$1 - local home_dir=$2 - - if ! id "$username" &>/dev/null; then - echo "Creating user: $username" - useradd -m -d "$home_dir" -s /bin/bash "$username" - - # Set a default password - echo "$username:$username" | chpasswd - - echo "User $username created successfully" - else - echo "User $username already exists" - fi -} - -# Create alice user -create_user_if_not_exists "alice" "/home/alice" -# Create abram user -create_user_if_not_exists "abram" "/home/abram" From 7a2304149cccceb5e07ec47d1cfa5c4aa2a6f179 Mon Sep 17 00:00:00 2001 From: Ramesh Mani Date: Wed, 8 Oct 2025 15:50:15 -0700 Subject: [PATCH 5/5] RANGER-5310:Include Apache Tez as the process framework for ranger-hive docker - address review comment on the switch user statement --- dev-support/ranger-docker/Dockerfile.ranger | 7 ------- dev-support/ranger-docker/Dockerfile.ranger-hadoop | 3 --- dev-support/ranger-docker/Dockerfile.ranger-hbase | 6 ------ dev-support/ranger-docker/Dockerfile.ranger-hive | 6 ------ dev-support/ranger-docker/Dockerfile.ranger-kafka | 6 ------ dev-support/ranger-docker/Dockerfile.ranger-kms | 3 --- dev-support/ranger-docker/Dockerfile.ranger-knox | 6 ------ dev-support/ranger-docker/Dockerfile.ranger-tagsync | 3 --- dev-support/ranger-docker/Dockerfile.ranger-usersync | 3 --- 9 files changed, 43 deletions(-) diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index 349cc7be09..b6dcff30a4 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -23,10 +23,6 @@ ARG RANGER_VERSION ARG RANGER_DB_TYPE ARG TARGETARCH -# Set Ranger environment variables -ENV RANGER_HOME=/opt/ranger -ENV RANGER_SCRIPTS=/home/ranger/scripts - COPY ./dist/version /home/ranger/dist/ COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/ @@ -34,9 +30,6 @@ COPY ./scripts/ranger.sh ${RANGER_SCRIPT COPY ./scripts/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-admin-install.properties COPY ./scripts/create-ranger-services.py ${RANGER_SCRIPTS}/ -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} \ && ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin \ && rm -f /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop index e036750d6c..8ae23dbc4d 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop +++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop @@ -36,9 +36,6 @@ COPY ./scripts/ranger-hadoop-mkdir.sh /home/ranger/scripts/ COPY ./scripts/ranger-hdfs-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/ranger-yarn-plugin-install.properties /home/ranger/scripts/ -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \ ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \ rm -f /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz && \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase index dccbc49442..0d01200796 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hbase +++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase @@ -31,9 +31,6 @@ COPY ./scripts/ranger-hbase.sh /home/ranger/scripts/ COPY ./scripts/ranger-hbase-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/hbase-site.xml /home/ranger/scripts/ -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \ ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \ rm -f /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz && \ @@ -51,7 +48,4 @@ RUN apt-get update && \ ENV HBASE_HOME=/opt/hbase ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hbase/bin -# Ensure container runs as root for SSH and service management -USER root - ENTRYPOINT [ "/home/ranger/scripts/ranger-hbase.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hive b/dev-support/ranger-docker/Dockerfile.ranger-hive index c562bafe42..6e8c4f2c9f 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-hive +++ b/dev-support/ranger-docker/Dockerfile.ranger-hive @@ -40,9 +40,6 @@ COPY ./scripts/ranger-hive.sh /home/ranger/scripts/ COPY ./scripts/ranger-hive-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/hive-site-${RANGER_DB_TYPE}.xml /home/ranger/scripts/hive-site.xml -# Switch to root to create system directories -USER root - RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \ rm -f /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ @@ -67,7 +64,4 @@ ENV TEZ_HOME=/opt/tez ENV TEZ_CONF_DIR=/opt/tez/conf ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin:/opt/tez/bin -# Ensure container runs as root for SSH and service management -USER root - ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka index 6dc9f59c21..922aeafa07 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kafka +++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka @@ -30,9 +30,6 @@ COPY ./scripts/ranger-kafka-setup.sh /home/ranger/scripts/ COPY ./scripts/ranger-kafka.sh /home/ranger/scripts/ COPY ./scripts/ranger-kafka-plugin-install.properties /home/ranger/scripts/ -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \ ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \ rm -f /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \ @@ -46,7 +43,4 @@ RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ ENV KAFKA_HOME=/opt/kafka ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin -# Ensure container runs as root for SSH and service management -USER root - ENTRYPOINT [ "/home/ranger/scripts/ranger-kafka.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms index 1758be9a32..604fe69222 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kms +++ b/dev-support/ranger-docker/Dockerfile.ranger-kms @@ -28,9 +28,6 @@ COPY ./dist/ranger-${KMS_VERSION}-kms.tar.gz /home/ranger/dist COPY ./scripts/ranger-kms.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-kms-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-kms-install.properties -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${KMS_VERSION}-kms ${RANGER_HOME}/kms && \ rm -f /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz && \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-knox b/dev-support/ranger-docker/Dockerfile.ranger-knox index e3365939e5..653af09ee2 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-knox +++ b/dev-support/ranger-docker/Dockerfile.ranger-knox @@ -32,9 +32,6 @@ COPY ./scripts/ranger-knox-plugin-install.properties /home/ranger/scripts/ COPY ./scripts/ranger-knox-expect.py /home/ranger/scripts/ COPY ./scripts/ranger-knox-sandbox.xml /home/ranger/scripts/ -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz --directory=/opt/ && \ ln -s /opt/knox-${KNOX_VERSION} /opt/knox && \ rm -f /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz && \ @@ -53,7 +50,4 @@ RUN chmod a+rwx /home/ranger/scripts/ranger-knox-expect.py RUN pip3 install pexpect RUN python3 /home/ranger/scripts/ranger-knox-expect.py -# Ensure container runs as root for SSH and service management -USER root - ENTRYPOINT [ "/home/ranger/scripts/ranger-knox.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-tagsync b/dev-support/ranger-docker/Dockerfile.ranger-tagsync index 352322f925..59efb40eb3 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-tagsync +++ b/dev-support/ranger-docker/Dockerfile.ranger-tagsync @@ -27,9 +27,6 @@ COPY ./scripts/ranger-tagsync.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-tagsync-install.properties ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-tagsync-tags.json ${RANGER_SCRIPTS}/ -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${TAGSYNC_VERSION}-tagsync ${RANGER_HOME}/tagsync && \ rm -f /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz && \ diff --git a/dev-support/ranger-docker/Dockerfile.ranger-usersync b/dev-support/ranger-docker/Dockerfile.ranger-usersync index 294c63c83d..af1d15c3d3 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-usersync +++ b/dev-support/ranger-docker/Dockerfile.ranger-usersync @@ -27,9 +27,6 @@ COPY ./scripts/ranger-usersync.sh ${RANGER_SCRIPTS}/ COPY ./scripts/ranger-usersync-install.properties ${RANGER_SCRIPTS}/ COPY ./scripts/ugsync-file-source.csv ${RANGER_SCRIPTS}/ -# Switch to root to create system directories -USER root - RUN tar xvfz /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz --directory=${RANGER_HOME} && \ ln -s ${RANGER_HOME}/ranger-${USERSYNC_VERSION}-usersync ${RANGER_HOME}/usersync && \ rm -f /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz && \