tests: test with Impala4 (#13)

Tests with Impala 4.4.1 using the github.com/apache/impala docker quickstart and apache/impala dockerhub images. The tests with Impala 3 remain with the apache/kudu:impala-latest image. apache/kudu doesn't deliver other impala version and I couldn't build them locally either. Closes #7
sclgo · Jan 6, 2025 · 377bc0b · 377bc0b
1 parent a2367f1
commit 377bc0b
Show file tree

Hide file tree

Showing 6 changed files with 1,078 additions and 80 deletions.
diff --git a/compose/quickstart-kudu-minimal.yml b/compose/quickstart-kudu-minimal.yml
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Minimal Apache Kudu cluster for quickstart with a single master and three tservers.
+# The Kudu repository contains a docker compose file (docker/quickstart.yml) for a more
+# production-like Kudu cluster with multiple masters, etc. This file is derived from that
+# file.
+version: "3"
+services:
+  kudu-master-1:
+    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
+    ports:
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:7051:7051"
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:8051:8051"
+    command: ["master"]
+    volumes:
+      - kudu-minimal-master-1:/var/lib/kudu
+    environment:
+      # TODO: Use `host.docker.internal` instead of QUICKSTART_IP when it
+      # works on Linux (https://github.com/docker/for-linux/issues/264)
+      - >
+        MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master
+        --rpc_bind_addresses=0.0.0.0:7051
+        --rpc_advertised_addresses=${QUICKSTART_IP:?Please set QUICKSTART_IP environment variable}:7051
+        --webserver_port=8051
+        --webserver_advertised_addresses=${QUICKSTART_IP}:8051
+        --webserver_doc_root=/opt/kudu/www
+        --stderrthreshold=0
+        --use_hybrid_clock=false
+        --memory_limit_hard_bytes=1073741824
+    networks:
+      - quickstart-network
+  kudu-tserver-1:
+    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
+    depends_on:
+      - kudu-master-1
+    ports:
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:7050:7050"
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:8050:8050"
+    command: ["tserver"]
+    volumes:
+      - kudu-minimal-tserver-1:/var/lib/kudu
+    environment:
+      - KUDU_MASTERS=kudu-master-1:7051
+      - >
+        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
+        --rpc_bind_addresses=0.0.0.0:7050
+        --rpc_advertised_addresses=${QUICKSTART_IP}:7050
+        --webserver_port=8050
+        --webserver_advertised_addresses=${QUICKSTART_IP}:8050
+        --webserver_doc_root=/opt/kudu/www
+        --stderrthreshold=0
+        --use_hybrid_clock=false
+        --memory_limit_hard_bytes=1073741824
+    networks:
+      - quickstart-network
+  kudu-tserver-2:
+    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
+    depends_on:
+      - kudu-master-1
+    ports:
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:7150:7150"
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:8150:8150"
+    command: ["tserver"]
+    volumes:
+      - kudu-minimal-tserver-2:/var/lib/kudu
+    environment:
+      - KUDU_MASTERS=kudu-master-1:7051
+      - >
+        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
+        --rpc_bind_addresses=0.0.0.0:7150
+        --rpc_advertised_addresses=${QUICKSTART_IP}:7150
+        --webserver_port=8150
+        --webserver_advertised_addresses=${QUICKSTART_IP}:8150
+        --webserver_doc_root=/opt/kudu/www
+        --stderrthreshold=0
+        --use_hybrid_clock=false
+        --memory_limit_hard_bytes=1073741824
+    networks:
+      - quickstart-network
+  kudu-tserver-3:
+    image: apache/kudu:${KUDU_QUICKSTART_VERSION:-latest}
+    depends_on:
+      - kudu-master-1
+    ports:
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:7250:7250"
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:8250:8250"
+    command: ["tserver"]
+    volumes:
+      - kudu-minimal-tserver-3:/var/lib/kudu
+    environment:
+      - KUDU_MASTERS=kudu-master-1:7051
+      - >
+        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
+        --rpc_bind_addresses=0.0.0.0:7250
+        --rpc_advertised_addresses=${QUICKSTART_IP}:7250
+        --webserver_port=8250
+        --webserver_advertised_addresses=${QUICKSTART_IP}:8250
+        --webserver_doc_root=/opt/kudu/www
+        --stderrthreshold=0
+        --use_hybrid_clock=false
+        --memory_limit_hard_bytes=1073741824
+    networks:
+      - quickstart-network
+volumes:
+  kudu-minimal-master-1:
+  kudu-minimal-tserver-1:
+  kudu-minimal-tserver-2:
+  kudu-minimal-tserver-3:
+
+networks:
+  quickstart-network:
+    external: true
+
diff --git a/compose/quickstart.yml b/compose/quickstart.yml
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This defines a basic Impala service with a single Impala Daemon and the minimal set of
+# services required to support it. A Hive MetaStore service is used to manage metadata.
+# All filesystem data is stored in Docker volumes. The default storage location for tables
+# is in the impala-quickstart-warehouse volume, i.e. if you create a table in Impala, it
+# will be stored in that volume by default.
+#
+# See README.md in this directory for usage instructions.
+version: "3"
+services:
+  hms:
+    image: ${IMPALA_QUICKSTART_IMAGE_PREFIX:-}impala_quickstart_hms
+    # container_name matches hostname in quickstart_conf/hive-site.xml
+    container_name: quickstart-hive-metastore
+    command: ["hms"]
+    volumes:
+      # Volume used to store Apache Derby database.
+      - impala-quickstart-warehouse:/var/lib/hive
+      # Warehouse directory. HMS does file operations so needs access to the
+      # shared volume.
+      - impala-quickstart-warehouse:/user/hive/warehouse
+      - ./quickstart_conf:/opt/hive/conf:ro
+    networks:
+      - quickstart-network
+  statestored:
+    image: ${IMPALA_QUICKSTART_IMAGE_PREFIX:-}statestored
+    ports:
+      # Web debug UI
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:25010:25010"
+    command: ["-redirect_stdout_stderr=false", "-logtostderr", "-v=1"]
+    volumes:
+      - ./quickstart_conf:/opt/impala/conf:ro
+    networks:
+      - quickstart-network
+  catalogd:
+    depends_on:
+      - statestored
+      - hms
+    image: ${IMPALA_QUICKSTART_IMAGE_PREFIX:-}catalogd
+    ports:
+      # Web debug UI
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:25020:25020"
+    command: ["-redirect_stdout_stderr=false", "-logtostderr", "-v=1",
+              "-hms_event_polling_interval_s=1", "-invalidate_tables_timeout_s=999999"]
+    volumes:
+      # Warehouse directory. Catalog does file operations so needs access to the
+      # shared volume.
+      - impala-quickstart-warehouse:/user/hive/warehouse
+      - ./quickstart_conf:/opt/impala/conf:ro
+    networks:
+      - quickstart-network
+  impalad-1:
+    image: ${IMPALA_QUICKSTART_IMAGE_PREFIX:-}impalad_coord_exec
+    depends_on:
+      - statestored
+      - catalogd
+    ports:
+      # Beeswax endpoint (deprecated)
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:21000:21000"
+      # HS2 endpoint
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:21050:21050"
+      # Web debug UI
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:25000:25000"
+      # HS2 over HTTP endpoint.
+      - "${QUICKSTART_LISTEN_ADDR:?Please set QUICKSTART_LISTEN_ADDR environment variable}:28000:28000"
+    command: [ "-v=1",
+              "-redirect_stdout_stderr=false", "-logtostderr",
+              "-kudu_master_hosts=kudu-master-1:7051",
+              "-mt_dop_auto_fallback=true",
+              "-default_query_options=mt_dop=4,default_file_format=parquet,default_transactional_type=insert_only",
+              "-mem_limit=4gb"]
+    environment:
+      # Keep the Java heap small to preserve memory for query execution.
+      - JAVA_TOOL_OPTIONS="-Xmx1g"
+    volumes:
+      - impala-quickstart-warehouse:/user/hive/warehouse
+      - ./quickstart_conf:/opt/impala/conf:ro
+    networks:
+      - quickstart-network
+volumes:
+  impala-quickstart-hms:
+  impala-quickstart-warehouse:
+
+networks:
+  quickstart-network:
+    # Workaround described in https://github.com/compose-spec/compose-go/pull/294#issuecomment-1455224674
+    name: quickstart-network
+
diff --git a/compose/quickstart_conf/hive-site.xml b/compose/quickstart_conf/hive-site.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<!--
+Hive configuration for Impala quickstart docker cluster.
+-->
+<configuration>
+      <property>
+        <!-- Required for automatic metadata sync. -->
+        <name>hive.metastore.dml.events</name>
+        <value>true</value>
+      </property>
+      <property>
+        <!-- User impala is not authorized to consume notifications by default, disable
+             authentication to work around this. -->
+         <name>hive.metastore.event.db.notification.api.auth</name>
+        <value>false</value>
+      </property>
+      <property>
+        <name>hive.metastore.uris</name>
+        <value>thrift://quickstart-hive-metastore:9083</value>
+      </property>
+      <!-- Managed and external tablespaces must live on the Docker volumes that we
+           configure for the quickstart cluster. -->
+      <property>
+        <name>hive.metastore.warehouse.dir</name>
+        <value>/user/hive/warehouse/managed</value>
+      </property>
+      <property>
+        <name>hive.metastore.warehouse.external.dir</name>
+        <value>/user/hive/warehouse/external</value>
+      </property>
+      <property>
+        <!-- Required to enable Hive transactions -->
+        <name>hive.support.concurrency</name>
+        <value>true</value>
+      </property>
+      <property>
+        <!-- Required to enable Hive transactions -->
+        <name>hive.txn.manager</name>
+        <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
+      </property>
+      <property>
+        <!-- Use embedded Derby database -->
+        <name>javax.jdo.option.ConnectionDriverName</name>
+        <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+      </property>
+      <property>
+        <!-- Use embedded Derby database -->
+        <name>javax.jdo.option.ConnectionURL</name>
+        <value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value>
+      </property>
+      <!-- Hive stats autogathering negatively affects latency of DDL operations, etc and
+           is not particularly useful for Impala -->
+      <property>
+        <name>hive.stats.autogather</name>
+        <value>false</value>
+      </property>
+</configuration>