RANGER-5310:Include Apache Tez as the process framework for ranger-hive docker

dev-support/ranger-docker/.env

@@ -11,46 +11,66 @@ RANGER_BASE_IMAGE=apache/ranger-base
 RANGER_BASE_VERSION=20250707-1-8
 # Java version used to build Apache Ranger is present as suffix: -8, valid values for suffix: -8, -11, -17
 RANGER_BASE_BUILD_VERSION=20250707-1-8
+RANGER_VERSION=3.0.0-SNAPSHOT
 
-# third party image versions
-MARIADB_VERSION=10.7.3
-POSTGRES_VERSION=12
-ORACLE_VERSION=23.6
-SQLSERVER_VERSION=2019-latest
-ENABLE_DB_MOUNT=true
-ZK_VERSION=3.9.2
-SOLR_VERSION=8.11.3
-
-# service versions
+# Hadoop Configuration
 HADOOP_VERSION=3.3.6
-HBASE_VERSION=2.6.0
+HDFS_PLUGIN_VERSION=3.0.0-SNAPSHOT
+YARN_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Hive Configuration
+HIVE_HADOOP_VERSION=3.3.6
 HIVE_VERSION=4.0.1
-HIVE_HADOOP_VERSION=3.1.1
+HIVE_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Tez Configuration
+TEZ_VERSION=0.10.4
+
+# HBase Configuration
+HBASE_VERSION=2.6.0
+HBASE_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Kafka Configuration
 KAFKA_VERSION=2.8.2
+KAFKA_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Knox Configuration
 KNOX_VERSION=2.0.0
-TRINO_VERSION=377
-OZONE_VERSION=1.4.0
-OZONE_RUNNER_VERSION=20230615-1
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
+KNOX_PLUGIN_VERSION=3.0.0-SNAPSHOT
 
-# versions of ranger services
-RANGER_VERSION=3.0.0-SNAPSHOT
+# KMS Configuration
 KMS_VERSION=3.0.0-SNAPSHOT
+
+# Usersync Configuration
 USERSYNC_VERSION=3.0.0-SNAPSHOT
+
+# Tagsync Configuration
 TAGSYNC_VERSION=3.0.0-SNAPSHOT
 
-# plugin versions
-HDFS_PLUGIN_VERSION=3.0.0-SNAPSHOT
-YARN_PLUGIN_VERSION=3.0.0-SNAPSHOT
-HIVE_PLUGIN_VERSION=3.0.0-SNAPSHOT
-HBASE_PLUGIN_VERSION=3.0.0-SNAPSHOT
-KAFKA_PLUGIN_VERSION=3.0.0-SNAPSHOT
-KNOX_PLUGIN_VERSION=3.0.0-SNAPSHOT
-TRINO_PLUGIN_VERSION=3.0.0-SNAPSHOT
+# Solr Configuration
+SOLR_VERSION=8.11.2
+
+# Zookeeper Configuration
+ZK_VERSION=3.8.4
+
+# Database Versions
+POSTGRES_VERSION=13.16
+MARIADB_VERSION=10.6
+ORACLE_VERSION=21.3.0-xe
+SQLSERVER_VERSION=2022-latest
+
+# Ozone Configuration
+OZONE_VERSION=1.4.0
 OZONE_PLUGIN_VERSION=3.0.0-SNAPSHOT
+OZONE_RUNNER_IMAGE=apache/ozone-runner
+OZONE_RUNNER_VERSION=20240625
+
+# Trino Configuration
+TRINO_VERSION=435
+TRINO_PLUGIN_VERSION=3.0.0-SNAPSHOT
 
-# To enable debug logs
+# Debug Configuration
 DEBUG_ADMIN=false
 DEBUG_USERSYNC=false
 DEBUG_TAGSYNC=false
+ENABLE_FILE_SYNC_SOURCE=false

dev-support/ranger-docker/Dockerfile.ranger-hadoop

@@ -21,12 +21,14 @@ FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
 ARG HADOOP_VERSION
 ARG HDFS_PLUGIN_VERSION
 ARG YARN_PLUGIN_VERSION
+ARG TEZ_VERSION
 
 
 COPY ./dist/version                                          /home/ranger/dist/
 COPY ./dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/
 COPY ./dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz /home/ranger/dist/
 COPY ./downloads/hadoop-${HADOOP_VERSION}.tar.gz             /home/ranger/dist/
+COPY ./downloads/apache-tez-${TEZ_VERSION}-bin.tar.gz        /home/ranger/dist/
 
 COPY ./scripts/ranger-hadoop-setup.sh                   /home/ranger/scripts/
 COPY ./scripts/ranger-hadoop.sh                         /home/ranger/scripts/
@@ -37,6 +39,8 @@ COPY ./scripts/ranger-yarn-plugin-install.properties    /home/ranger/scripts/
 RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \
     ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \
     rm -f /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz && \
+    tar xvfz /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz --directory=/opt/ && \
+    ln -s /opt/apache-tez-${TEZ_VERSION}-bin /opt/tez && \
     tar xvfz /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin /opt/ranger/ranger-hdfs-plugin && \
     rm -f /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz && \

dev-support/ranger-docker/Dockerfile.ranger-hive

@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG RANGER_DB_TYPE
+# Only the ARGs needed for this build stage
 
 ARG RANGER_BASE_IMAGE
 ARG RANGER_BASE_VERSION
@@ -22,13 +22,15 @@ FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
 ARG HIVE_VERSION
 ARG HIVE_HADOOP_VERSION
 ARG HIVE_PLUGIN_VERSION
+ARG TEZ_VERSION
 ARG RANGER_DB_TYPE
 
 
 COPY ./dist/version                                          /home/ranger/dist/
 COPY ./dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz /home/ranger/dist/
 COPY ./downloads/apache-hive-${HIVE_VERSION}-bin.tar.gz      /home/ranger/dist/
 COPY ./downloads/hadoop-${HIVE_HADOOP_VERSION}.tar.gz        /home/ranger/dist/
+COPY ./downloads/apache-tez-${TEZ_VERSION}-bin.tar.gz        /home/ranger/dist/
 COPY ./downloads/postgresql-42.2.16.jre7.jar                 /home/ranger/dist/
 COPY ./downloads/mysql-connector-java-8.0.28.jar             /home/ranger/dist/
 COPY ./downloads/ojdbc8.jar                                  /home/ranger/dist/
@@ -38,7 +40,7 @@ COPY ./scripts/ranger-hive.sh                           /home/ranger/scripts/
 COPY ./scripts/ranger-hive-plugin-install.properties    /home/ranger/scripts/
 COPY ./scripts/hive-site-${RANGER_DB_TYPE}.xml          /home/ranger/scripts/hive-site.xml
 
-RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \
+RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
     ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
     rm -f /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
     mv /home/ranger/dist/postgresql-42.2.16.jre7.jar /opt/hive/lib/ && \
@@ -47,6 +49,9 @@ RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --director
     tar xvfz /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --directory=/opt/ && \
     ln -s /opt/hadoop-${HIVE_HADOOP_VERSION} /opt/hadoop && \
     rm -f /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \
+    tar xvfz /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz --directory=/opt/ && \
+    ln -s /opt/apache-tez-${TEZ_VERSION}-bin /opt/tez && \
+    rm -f /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz && \
     tar xvfz /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \
     rm -f /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz && \
@@ -55,7 +60,8 @@ RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --director
 
 ENV HIVE_HOME=/opt/hive
 ENV HADOOP_HOME=/opt/hadoop
-ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin
-
+ENV TEZ_HOME=/opt/tez
+ENV TEZ_CONF_DIR=/opt/tez/conf
+ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin:/opt/tez/bin
 
 ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ]

dev-support/ranger-docker/Dockerfile.ranger-kafka

@@ -39,6 +39,7 @@ RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/
     cp -f /home/ranger/scripts/ranger-kafka-plugin-install.properties /opt/ranger/ranger-kafka-plugin/install.properties && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-kafka-setup.sh ${RANGER_SCRIPTS}/ranger-kafka.sh
 
+
 ENV KAFKA_HOME=/opt/kafka
 ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
 

dev-support/ranger-docker/Dockerfile.ranger-kms

@@ -13,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 ARG RANGER_DB_TYPE
 ARG RANGER_BASE_IMAGE
 ARG RANGER_BASE_VERSION
@@ -38,7 +39,7 @@ RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RA
     ln -s /etc/init.d/ranger-kms /etc/rc3.d/S88ranger-kms && \
     ln -s /etc/init.d/ranger-kms /etc/rc3.d/K90ranger-kms && \
     ln -s ${RANGER_HOME}/kms/ranger-kms-services.sh /usr/bin/ranger-kms-services.sh && \
-    chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ && \
+    chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ /etc/ranger && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-kms.sh
 
 FROM ranger-kms AS ranger_postgres

dev-support/ranger-docker/Dockerfile.ranger-usersync

@@ -31,10 +31,9 @@ RUN tar xvfz /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz --dire
     ln -s ${RANGER_HOME}/ranger-${USERSYNC_VERSION}-usersync ${RANGER_HOME}/usersync && \
     rm -f /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz && \
     cp -f ${RANGER_SCRIPTS}/ranger-usersync-install.properties ${RANGER_HOME}/usersync/install.properties && \
-    mkdir -p /var/run/ranger /var/log/ranger/usersync /etc/ranger && \
-    mkdir /etc/init.d || true && \
-    mkdir /etc/rc2.d  || true && \
-    mkdir /etc/rc3.d  || true && \
+    mkdir -p /var/run/ranger /var/log/ranger/usersync && \
+    mkdir -p /etc/ranger && \
+    mkdir -p /etc/init.d /etc/rc2.d /etc/rc3.d && \
     touch /etc/init.d/ranger-usersync && \
     ln -s /etc/init.d/ranger-usersync /etc/rc2.d/S99ranger-usersync && \
     ln -s /etc/init.d/ranger-usersync /etc/rc2.d/K00ranger-usersync && \

dev-support/ranger-docker/config/my.cnf

@@ -19,5 +19,7 @@
 collation-server = utf8_unicode_ci
 init-connect = 'SET NAMES utf8'
 character_set_server = utf8
+# Enable hostname resolution for user authentication (fixes ranger-db user issue)
+skip-name-resolve = 0
 # transaction-isolation = REPEATABLE-READ
 # transaction-isolation = READ-COMMITTED

dev-support/ranger-docker/docker-compose.ranger-db.yml

@@ -25,7 +25,7 @@ services:
       args:
         - MARIADB_VERSION=${MARIADB_VERSION}
     image: ranger-mysql
-    command: --default-authentication-plugin=mysql_native_password
+    command: --default-authentication-plugin=mysql_native_password --skip-name-resolve=0
     container_name: ranger-mysql
     hostname: ranger-db.example.com
     ports:

dev-support/ranger-docker/docker-compose.ranger-hadoop.yml

@@ -9,6 +9,7 @@ services:
         - HADOOP_VERSION=${HADOOP_VERSION}
         - HDFS_PLUGIN_VERSION=${HDFS_PLUGIN_VERSION}
         - YARN_PLUGIN_VERSION=${YARN_PLUGIN_VERSION}
+        - TEZ_VERSION=${TEZ_VERSION}
     image: ranger-hadoop
     container_name: ranger-hadoop
     hostname: ranger-hadoop.example.com
@@ -32,6 +33,7 @@ services:
       - HADOOP_VERSION
       - HDFS_PLUGIN_VERSION
       - YARN_PLUGIN_VERSION
+      - TEZ_VERSION
 
 networks:
   ranger:

dev-support/ranger-docker/docker-compose.ranger-hive.yml

@@ -9,6 +9,7 @@ services:
         - HIVE_HADOOP_VERSION=${HIVE_HADOOP_VERSION}
         - HIVE_VERSION=${HIVE_VERSION}
         - HIVE_PLUGIN_VERSION=${HIVE_PLUGIN_VERSION}
+        - TEZ_VERSION=${TEZ_VERSION}
         - RANGER_DB_TYPE=${RANGER_DB_TYPE}
     image: ranger-hive
     container_name: ranger-hive
@@ -19,6 +20,7 @@ services:
       - ranger
     ports:
       - "10000:10000"
+      - "9083:9083"
     depends_on:
       ranger:
         condition: service_started
@@ -30,6 +32,7 @@ services:
       - HIVE_HADOOP_VERSION
       - HIVE_VERSION
       - HIVE_PLUGIN_VERSION
+      - TEZ_VERSION
       - RANGER_DB_TYPE
 
 networks:

dev-support/ranger-docker/download-archives.sh

@@ -53,6 +53,7 @@ then
     downloadIfNotPresent hbase-${HBASE_VERSION}-bin.tar.gz      https://archive.apache.org/dist/hbase/${HBASE_VERSION}
     downloadIfNotPresent apache-hive-${HIVE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}
     downloadIfNotPresent hadoop-${HIVE_HADOOP_VERSION}.tar.gz   https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION}
+    downloadIfNotPresent apache-tez-${TEZ_VERSION}-bin.tar.gz   https://archive.apache.org/dist/tez/${TEZ_VERSION}
     downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz        https://archive.apache.org/dist/kafka/${KAFKA_VERSION}
     downloadIfNotPresent knox-${KNOX_VERSION}.tar.gz            https://archive.apache.org/dist/knox/${KNOX_VERSION}
     downloadIfNotPresent ozone-${OZONE_VERSION}.tar.gz          https://archive.apache.org/dist/ozone/${OZONE_VERSION}
@@ -72,6 +73,7 @@ else
     then
       downloadIfNotPresent apache-hive-${HIVE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}
       downloadIfNotPresent hadoop-${HIVE_HADOOP_VERSION}.tar.gz   https://archive.apache.org/dist/hadoop/common/hadoop-${HIVE_HADOOP_VERSION}
+      downloadIfNotPresent apache-tez-${TEZ_VERSION}-bin.tar.gz   https://archive.apache.org/dist/tez/${TEZ_VERSION}
     elif [[ $arg == 'kafka' ]]
     then
       downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz        https://archive.apache.org/dist/kafka/${KAFKA_VERSION}

dev-support/ranger-docker/scripts/hive-site-mysql.xml

@@ -49,4 +49,72 @@
         <name>hive.zookeeper.client.port</name>
         <value>2181</value>
     </property>
+
+    <!-- Tez Configuration -->
+    <property>
+        <name>hive.execution.engine</name>
+        <value>tez</value>
+        <description>Execution engine to use for Hive queries</description>
+    </property>
+
+    <property>
+        <name>hive.tez.container.size</name>
+        <value>1024</value>
+        <description>By default Tez will spawn containers of the size of a mapper</description>
+    </property>
+
+    <property>
+        <name>hive.tez.java.opts</name>
+        <value>-Xmx768m</value>
+        <description>Java command line options for Tez</description>
+    </property>
+
+    <property>
+        <name>hive.tez.log.level</name>
+        <value>INFO</value>
+        <description>Log level for Tez</description>
+    </property>
+
+    <property>
+        <name>hive.prewarm.enabled</name>
+        <value>false</value>
+        <description>Enables container prewarm for Tez</description>
+    </property>
+
+    <property>
+        <name>hive.prewarm.numcontainers</name>
+        <value>3</value>
+        <description>Controls the number of containers to prewarm for Tez</description>
+    </property>
+
+    <property>
+        <name>hive.tez.auto.reducer.parallelism</name>
+        <value>true</value>
+        <description>Turn on Tez' auto reducer parallelism feature</description>
+    </property>
+
+    <property>
+        <name>hive.tez.min.reducer.per.query</name>
+        <value>2</value>
+        <description>Controls the minimum number of reducers for Tez</description>
+    </property>
+
+    <property>
+        <name>hive.tez.max.reducer.per.query</name>
+        <value>999</value>
+        <description>Controls the maximum number of reducers for Tez</description>
+    </property>
+
+    <!-- YARN Configuration for Tez -->
+    <property>
+        <name>hive.tez.exec.print.summary</name>
+        <value>true</value>
+        <description>Whether to print a summary of the plan execution</description>
+    </property>
+
+    <property>
+        <name>tez.queue.name</name>
+        <value>default</value>
+        <description>YARN queue name for Tez jobs</description>
+    </property>
 </configuration>