From 5d3f8fc1205084343326af8448dbfb7f194a66a7 Mon Sep 17 00:00:00 2001 From: maxime1907 <19607336+maxime1907@users.noreply.github.com> Date: Thu, 16 May 2024 18:31:12 +0200 Subject: [PATCH] chore(conf): use upsteam conf --- .github/workflows/build_release.yml | 2 + Containerfile | 15 +- README.md | 35 ++-- conf/druid/.DS_Store | Bin 6148 -> 0 bytes conf/druid/auto/_common/common.jvm.config | 7 + .../auto/_common/common.runtime.properties | 161 ++++++++++++++++++ conf/druid/auto/_common/log4j2.xml | 86 ++++++++++ conf/druid/auto/broker/runtime.properties | 41 +++++ .../coordinator-overlord/runtime.properties | 33 ++++ conf/druid/auto/historical/runtime.properties | 35 ++++ conf/druid/auto/indexer/runtime.properties | 34 ++++ .../auto/middleManager/runtime.properties | 36 ++++ conf/druid/auto/router/runtime.properties | 28 +++ .../cluster/_common/common.runtime.properties | 158 +++++++++++++++++ conf/druid/cluster/_common/log4j2.xml | 91 ++++++++++ conf/druid/cluster/data/historical/jvm.config | 9 + .../druid/cluster/data/historical/main.config | 1 + .../data/historical/runtime.properties | 39 +++++ conf/druid/cluster/data/indexer/jvm.config | 9 + conf/druid/cluster/data/indexer/main.config | 1 + .../cluster/data/indexer/runtime.properties | 38 +++++ .../cluster/data/middleManager/jvm.config | 8 + .../cluster/data/middleManager/main.config | 1 + .../data/middleManager/runtime.properties | 40 +++++ .../master/coordinator-overlord/jvm.config | 10 ++ .../master/coordinator-overlord/main.config | 1 + .../coordinator-overlord/runtime.properties | 33 ++++ conf/druid/cluster/query/broker/jvm.config | 9 + conf/druid/cluster/query/broker/main.config | 1 + .../cluster/query/broker/runtime.properties | 37 ++++ conf/druid/cluster/query/router/jvm.config | 10 ++ conf/druid/cluster/query/router/main.config | 1 + .../cluster/query/router/runtime.properties | 34 ++++ conf/druid/single-server/.DS_Store | Bin 6148 -> 0 bytes .../large/_common/common.runtime.properties | 158 +++++++++++++++++ .../single-server/large/_common/log4j2.xml | 86 ++++++++++ .../single-server/large/broker/jvm.config | 10 ++ .../single-server/large/broker/main.config | 1 + .../large/broker/runtime.properties | 37 ++++ .../large/coordinator-overlord/jvm.config | 10 ++ .../large/coordinator-overlord/main.config | 1 + .../coordinator-overlord/runtime.properties | 33 ++++ .../single-server/large/historical/jvm.config | 10 ++ .../large/historical/main.config | 1 + .../large/historical/runtime.properties | 39 +++++ .../large/middleManager/jvm.config | 9 + .../large/middleManager/main.config | 1 + .../large/middleManager/runtime.properties | 40 +++++ .../single-server/large/router/jvm.config | 10 ++ .../single-server/large/router/main.config | 1 + .../large/router/runtime.properties | 34 ++++ .../medium/_common/common.runtime.properties | 158 +++++++++++++++++ .../single-server/medium/_common/log4j2.xml | 86 ++++++++++ .../single-server/medium/broker/jvm.config | 10 ++ .../single-server/medium/broker/main.config | 1 + .../medium/broker/runtime.properties | 37 ++++ .../medium/coordinator-overlord/jvm.config | 10 ++ .../medium/coordinator-overlord/main.config | 1 + .../coordinator-overlord/runtime.properties | 33 ++++ .../medium/historical/jvm.config | 10 ++ .../medium/historical/main.config | 1 + .../medium/historical/runtime.properties | 39 +++++ .../medium/middleManager/jvm.config | 9 + .../medium/middleManager/main.config | 1 + .../medium/middleManager/runtime.properties | 40 +++++ .../single-server/medium/router/jvm.config | 10 ++ .../single-server/medium/router/main.config | 1 + .../medium/router/runtime.properties | 34 ++++ .../_common/common.runtime.properties | 34 ++-- .../micro-quickstart/_common/log4j2.xml | 40 ++++- .../broker/runtime.properties | 7 +- .../coordinator-overlord/runtime.properties | 10 +- .../historical/runtime.properties | 7 +- .../middleManager/runtime.properties | 9 +- .../router/runtime.properties | 15 +- .../single-server/nano-quickstart/.DS_Store | Bin 6148 -> 0 bytes .../nano-quickstart/.idea/.gitignore | 8 - .../inspectionProfiles/Project_Default.xml | 58 ------- .../inspectionProfiles/profiles_settings.xml | 6 - .../nano-quickstart/.idea/misc.xml | 4 - .../nano-quickstart/.idea/modules.xml | 8 - .../nano-quickstart/.idea/nano-quickstart.iml | 8 - .../_common/common.runtime.properties | 34 ++-- .../nano-quickstart/_common/log4j2.xml | 42 ++++- .../nano-quickstart/broker/runtime.properties | 7 +- .../coordinator-overlord/runtime.properties | 10 +- .../historical/runtime.properties | 7 +- .../middleManager/runtime.properties | 9 +- .../nano-quickstart/router/runtime.properties | 12 +- .../small/_common/common.runtime.properties | 158 +++++++++++++++++ .../single-server/small/_common/log4j2.xml | 86 ++++++++++ .../single-server/small/broker/jvm.config | 10 ++ .../single-server/small/broker/main.config | 1 + .../small/broker/runtime.properties | 37 ++++ .../small/coordinator-overlord/jvm.config | 10 ++ .../small/coordinator-overlord/main.config | 1 + .../coordinator-overlord/runtime.properties | 33 ++++ .../single-server/small/historical/jvm.config | 10 ++ .../small/historical/main.config | 1 + .../small/historical/runtime.properties | 39 +++++ .../small/middleManager/jvm.config | 9 + .../small/middleManager/main.config | 1 + .../small/middleManager/runtime.properties | 40 +++++ .../single-server/small/router/jvm.config | 11 ++ .../single-server/small/router/main.config | 1 + .../small/router/runtime.properties | 34 ++++ .../xlarge/_common/common.runtime.properties | 158 +++++++++++++++++ .../single-server/xlarge/_common/log4j2.xml | 86 ++++++++++ .../single-server/xlarge/broker/jvm.config | 10 ++ .../single-server/xlarge/broker/main.config | 1 + .../xlarge/broker/runtime.properties | 37 ++++ .../xlarge/coordinator-overlord/jvm.config | 10 ++ .../xlarge/coordinator-overlord/main.config | 1 + .../coordinator-overlord/runtime.properties | 33 ++++ .../xlarge/historical/jvm.config | 10 ++ .../xlarge/historical/main.config | 1 + .../xlarge/historical/runtime.properties | 39 +++++ .../xlarge/middleManager/jvm.config | 9 + .../xlarge/middleManager/main.config | 1 + .../xlarge/middleManager/runtime.properties | 40 +++++ .../single-server/xlarge/router/jvm.config | 10 ++ .../single-server/xlarge/router/main.config | 1 + .../xlarge/router/runtime.properties | 34 ++++ conf/supervise/cluster/data.conf | 4 + conf/supervise/cluster/master-no-zk.conf | 3 + conf/supervise/cluster/master-with-zk.conf | 4 + conf/supervise/cluster/query.conf | 4 + conf/supervise/single-server/large.conf | 11 ++ conf/supervise/single-server/medium.conf | 11 ++ .../single-server/micro-quickstart.conf | 1 + .../single-server/nano-quickstart.conf | 1 + conf/supervise/single-server/small.conf | 11 ++ conf/supervise/single-server/xlarge.conf | 11 ++ conf/zk/log4j2.xml | 45 +++-- ...re-conf-extensions-postgresql-and-s3.patch | 39 +++++ ...f-enable-s3-instead-of-local-storage.patch | 125 ++++++++++++++ ...f-enable-s3-instead-of-local-storage.patch | 68 ++++++++ ...re-conf-extensions-postgresql-and-s3.patch | 25 +++ ...art-druid-autodetect-cpu-count-in-co.patch | 110 ++++++++++++ scripts/download_upstream_conf.sh | 15 ++ 140 files changed, 3564 insertions(+), 228 deletions(-) delete mode 100644 conf/druid/.DS_Store create mode 100644 conf/druid/auto/_common/common.jvm.config create mode 100644 conf/druid/auto/_common/common.runtime.properties create mode 100644 conf/druid/auto/_common/log4j2.xml create mode 100644 conf/druid/auto/broker/runtime.properties create mode 100644 conf/druid/auto/coordinator-overlord/runtime.properties create mode 100644 conf/druid/auto/historical/runtime.properties create mode 100644 conf/druid/auto/indexer/runtime.properties create mode 100644 conf/druid/auto/middleManager/runtime.properties create mode 100644 conf/druid/auto/router/runtime.properties create mode 100644 conf/druid/cluster/_common/common.runtime.properties create mode 100644 conf/druid/cluster/_common/log4j2.xml create mode 100644 conf/druid/cluster/data/historical/jvm.config create mode 100644 conf/druid/cluster/data/historical/main.config create mode 100644 conf/druid/cluster/data/historical/runtime.properties create mode 100644 conf/druid/cluster/data/indexer/jvm.config create mode 100644 conf/druid/cluster/data/indexer/main.config create mode 100644 conf/druid/cluster/data/indexer/runtime.properties create mode 100644 conf/druid/cluster/data/middleManager/jvm.config create mode 100644 conf/druid/cluster/data/middleManager/main.config create mode 100644 conf/druid/cluster/data/middleManager/runtime.properties create mode 100644 conf/druid/cluster/master/coordinator-overlord/jvm.config create mode 100644 conf/druid/cluster/master/coordinator-overlord/main.config create mode 100644 conf/druid/cluster/master/coordinator-overlord/runtime.properties create mode 100644 conf/druid/cluster/query/broker/jvm.config create mode 100644 conf/druid/cluster/query/broker/main.config create mode 100644 conf/druid/cluster/query/broker/runtime.properties create mode 100644 conf/druid/cluster/query/router/jvm.config create mode 100644 conf/druid/cluster/query/router/main.config create mode 100644 conf/druid/cluster/query/router/runtime.properties delete mode 100644 conf/druid/single-server/.DS_Store create mode 100644 conf/druid/single-server/large/_common/common.runtime.properties create mode 100644 conf/druid/single-server/large/_common/log4j2.xml create mode 100644 conf/druid/single-server/large/broker/jvm.config create mode 100644 conf/druid/single-server/large/broker/main.config create mode 100644 conf/druid/single-server/large/broker/runtime.properties create mode 100644 conf/druid/single-server/large/coordinator-overlord/jvm.config create mode 100644 conf/druid/single-server/large/coordinator-overlord/main.config create mode 100644 conf/druid/single-server/large/coordinator-overlord/runtime.properties create mode 100644 conf/druid/single-server/large/historical/jvm.config create mode 100644 conf/druid/single-server/large/historical/main.config create mode 100644 conf/druid/single-server/large/historical/runtime.properties create mode 100644 conf/druid/single-server/large/middleManager/jvm.config create mode 100644 conf/druid/single-server/large/middleManager/main.config create mode 100644 conf/druid/single-server/large/middleManager/runtime.properties create mode 100644 conf/druid/single-server/large/router/jvm.config create mode 100644 conf/druid/single-server/large/router/main.config create mode 100644 conf/druid/single-server/large/router/runtime.properties create mode 100644 conf/druid/single-server/medium/_common/common.runtime.properties create mode 100644 conf/druid/single-server/medium/_common/log4j2.xml create mode 100644 conf/druid/single-server/medium/broker/jvm.config create mode 100644 conf/druid/single-server/medium/broker/main.config create mode 100644 conf/druid/single-server/medium/broker/runtime.properties create mode 100644 conf/druid/single-server/medium/coordinator-overlord/jvm.config create mode 100644 conf/druid/single-server/medium/coordinator-overlord/main.config create mode 100644 conf/druid/single-server/medium/coordinator-overlord/runtime.properties create mode 100644 conf/druid/single-server/medium/historical/jvm.config create mode 100644 conf/druid/single-server/medium/historical/main.config create mode 100644 conf/druid/single-server/medium/historical/runtime.properties create mode 100644 conf/druid/single-server/medium/middleManager/jvm.config create mode 100644 conf/druid/single-server/medium/middleManager/main.config create mode 100644 conf/druid/single-server/medium/middleManager/runtime.properties create mode 100644 conf/druid/single-server/medium/router/jvm.config create mode 100644 conf/druid/single-server/medium/router/main.config create mode 100644 conf/druid/single-server/medium/router/runtime.properties delete mode 100644 conf/druid/single-server/nano-quickstart/.DS_Store delete mode 100644 conf/druid/single-server/nano-quickstart/.idea/.gitignore delete mode 100644 conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/Project_Default.xml delete mode 100644 conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/profiles_settings.xml delete mode 100644 conf/druid/single-server/nano-quickstart/.idea/misc.xml delete mode 100644 conf/druid/single-server/nano-quickstart/.idea/modules.xml delete mode 100644 conf/druid/single-server/nano-quickstart/.idea/nano-quickstart.iml create mode 100644 conf/druid/single-server/small/_common/common.runtime.properties create mode 100644 conf/druid/single-server/small/_common/log4j2.xml create mode 100644 conf/druid/single-server/small/broker/jvm.config create mode 100644 conf/druid/single-server/small/broker/main.config create mode 100644 conf/druid/single-server/small/broker/runtime.properties create mode 100644 conf/druid/single-server/small/coordinator-overlord/jvm.config create mode 100644 conf/druid/single-server/small/coordinator-overlord/main.config create mode 100644 conf/druid/single-server/small/coordinator-overlord/runtime.properties create mode 100644 conf/druid/single-server/small/historical/jvm.config create mode 100644 conf/druid/single-server/small/historical/main.config create mode 100644 conf/druid/single-server/small/historical/runtime.properties create mode 100644 conf/druid/single-server/small/middleManager/jvm.config create mode 100644 conf/druid/single-server/small/middleManager/main.config create mode 100644 conf/druid/single-server/small/middleManager/runtime.properties create mode 100644 conf/druid/single-server/small/router/jvm.config create mode 100644 conf/druid/single-server/small/router/main.config create mode 100644 conf/druid/single-server/small/router/runtime.properties create mode 100644 conf/druid/single-server/xlarge/_common/common.runtime.properties create mode 100644 conf/druid/single-server/xlarge/_common/log4j2.xml create mode 100644 conf/druid/single-server/xlarge/broker/jvm.config create mode 100644 conf/druid/single-server/xlarge/broker/main.config create mode 100644 conf/druid/single-server/xlarge/broker/runtime.properties create mode 100644 conf/druid/single-server/xlarge/coordinator-overlord/jvm.config create mode 100644 conf/druid/single-server/xlarge/coordinator-overlord/main.config create mode 100644 conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties create mode 100644 conf/druid/single-server/xlarge/historical/jvm.config create mode 100644 conf/druid/single-server/xlarge/historical/main.config create mode 100644 conf/druid/single-server/xlarge/historical/runtime.properties create mode 100644 conf/druid/single-server/xlarge/middleManager/jvm.config create mode 100644 conf/druid/single-server/xlarge/middleManager/main.config create mode 100644 conf/druid/single-server/xlarge/middleManager/runtime.properties create mode 100644 conf/druid/single-server/xlarge/router/jvm.config create mode 100644 conf/druid/single-server/xlarge/router/main.config create mode 100644 conf/druid/single-server/xlarge/router/runtime.properties create mode 100644 conf/supervise/cluster/data.conf create mode 100644 conf/supervise/cluster/master-no-zk.conf create mode 100644 conf/supervise/cluster/master-with-zk.conf create mode 100644 conf/supervise/cluster/query.conf create mode 100644 conf/supervise/single-server/large.conf create mode 100644 conf/supervise/single-server/medium.conf create mode 100644 conf/supervise/single-server/small.conf create mode 100644 conf/supervise/single-server/xlarge.conf create mode 100644 patches/conf/0001-chore-conf-extensions-postgresql-and-s3.patch create mode 100644 patches/conf/0002-chore-conf-enable-s3-instead-of-local-storage.patch create mode 100644 patches/conf/0003-chore-conf-enable-s3-instead-of-local-storage.patch create mode 100644 patches/conf/0004-chore-conf-extensions-postgresql-and-s3.patch create mode 100644 patches/druid/0001-feat-examples-start-druid-autodetect-cpu-count-in-co.patch create mode 100755 scripts/download_upstream_conf.sh diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml index c870db2..c9b36ed 100644 --- a/.github/workflows/build_release.yml +++ b/.github/workflows/build_release.yml @@ -24,6 +24,8 @@ jobs: os: - ubuntu-22.04 context: + - target: druid-auto + suffix: -auto - target: druid-micro suffix: -micro - target: druid-nano diff --git a/Containerfile b/Containerfile index 582d9ae..b1eae9b 100644 --- a/Containerfile +++ b/Containerfile @@ -1,10 +1,13 @@ -FROM openjdk:8-jre AS builder +FROM eclipse-temurin:11 AS builder ARG DRUID_VERSION=26.0.0 ENV DRUID_VERSION=$DRUID_VERSION RUN apt-get update \ - && apt-get install -y --no-install-recommends bash=5.1-2+deb11u1 perl=5.32.1-4+deb11u2 \ + && apt-get install -y --no-install-recommends \ + git=1:2.34* \ + libfindbin-libs-perl=2* \ + python3=3.10* \ && rm -rf /var/lib/apt/lists/* RUN wget --progress=dot:giga http://apache.crihan.fr/dist/druid/${DRUID_VERSION}/apache-druid-${DRUID_VERSION}-bin.tar.gz \ @@ -17,6 +20,10 @@ RUN mv /apache-druid-${DRUID_VERSION} /app WORKDIR /app +COPY patches/druid /tmp/patches + +RUN git apply /tmp/patches/* + FROM builder as druid-micro ENTRYPOINT ["/app/bin/start-micro-quickstart"] @@ -25,4 +32,8 @@ FROM builder as druid-nano ENTRYPOINT ["/app/bin/start-nano-quickstart"] +FROM builder as druid-auto + +ENTRYPOINT ["/app/bin/start-druid"] + EXPOSE 8081 8082 8083 8888 8091 diff --git a/README.md b/README.md index 7d9c1c0..0eae487 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,13 @@ -Micro & Nano druid, for E2E tests -======================================= +# Micro & Nano druid, for E2E tests Dockerfiles to run nano and micro setups of [druid](https://github.com/apache/druid) for e2e tests. Some configurations are applied to maximize the response time of the Druid docker but should in no way refer to a production cluster. -Arguments ---------- +## Arguments Arguments that can be set while building: - `DRUID_VERSION`: You can also change the default value in the Containerfile. -Building --------- +## Building ``` export DRUID_VERSION=XXX # 26.0.0 for example. @@ -22,22 +19,14 @@ docker build -t micro-druid:${DRUID_VERSION}-micro --build-arg DRUID_VERSION=${D docker build -t micro-druid:${DRUID_VERSION}-nano --build-arg DRUID_VERSION=${DRUID_VERSION} -f ./Containerfile --target druid-nano . ``` -Required hardware ------------------- +## Configuration +The `conf` folder must stay the same as the upstream one that can be found here: +- https://github.com/apache/druid/tree/26.0.1/examples/conf -Nano -********* - -* 1 CPU -* 4GB RAM -* Launch command: `bin/start-nano-quickstart` -* Configuration directory: `conf/druid/single-server/nano-quickstart` - - -Micro -********* +Run this script when updating the druid version: +```bash +./scripts/download_upstream_conf.sh +``` -* 4 CPU -* 16GB RAM -* Launch command: `bin/start-micro-quickstart` -* Configuration directory: `conf/druid/single-server/micro-quickstart` +## Required hardware +- https://github.com/apache/druid/blob/26.0.1/docs/operations/single-server.md diff --git a/conf/druid/.DS_Store b/conf/druid/.DS_Store deleted file mode 100644 index a58b880066534e446db3b074bdbbb3f6de0007f2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKF-`+P474FdBorwn<$i#IAFQI}1$+QO5JEZ;iHOu!@h+amjP2%h(1j)%OWxV_ zdVIPm&Sx|8&FB8!Y;9&sIMKei%#HW-nO#-HfpF9q?8WZvi1lNZ{d<7i6&P>!Zm$#m ziBuL0#ZNdW{&Kbi`Zb^};bR>E<bFQY?}=Ka zfE1W2aGl3B`~MF9$NWDf=_CcDz(pzGi|xa9gIB8FI(s?x+6I4wGv^yl!#*fjq8$UH i9pk`ud>cub*SN=hFB}ttj(E_4`Wc`uGAZ!i3j6>Zsup|z diff --git a/conf/druid/auto/_common/common.jvm.config b/conf/druid/auto/_common/common.jvm.config new file mode 100644 index 0000000..fd74cf3 --- /dev/null +++ b/conf/druid/auto/_common/common.jvm.config @@ -0,0 +1,7 @@ +-server +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/auto/_common/common.runtime.properties b/conf/druid/auto/_common/common.runtime.properties new file mode 100644 index 0000000..a841ff7 --- /dev/null +++ b/conf/druid/auto/_common/common.runtime.properties @@ -0,0 +1,161 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "postgresql-metadata-storage", "druid-s3-extensions"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +# druid.storage.type=local +# druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +druid.storage.type=s3 +druid.storage.bucket=storage-bucket +druid.storage.baseKey=druid/segments +druid.s3.accessKey=access_key +druid.s3.secretKey=secret_key + +druid.s3.endpoint.url=http://localhost:9000 +druid.s3.enablePathStyleAccess=true + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +# druid.indexer.logs.type=file +# druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +druid.indexer.logs.type=s3 +druid.indexer.logs.s3Bucket=logs-bucket +druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/auto/_common/log4j2.xml b/conf/druid/auto/_common/log4j2.xml new file mode 100644 index 0000000..66dc13d --- /dev/null +++ b/conf/druid/auto/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/druid/auto/broker/runtime.properties b/conf/druid/auto/broker/runtime.properties new file mode 100644 index 0000000..f4c4940 --- /dev/null +++ b/conf/druid/auto/broker/runtime.properties @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N + +# HTTP client settings +# Connection pool size from the Broker to each data server. May be useful to +# raise this for high-concurrency workloads. +# druid.broker.http.numConnections=20 + +# Processing threads and buffers +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/conf/druid/auto/coordinator-overlord/runtime.properties b/conf/druid/auto/coordinator-overlord/runtime.properties new file mode 100644 index 0000000..c053823 --- /dev/null +++ b/conf/druid/auto/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/conf/druid/auto/historical/runtime.properties b/conf/druid/auto/historical/runtime.properties new file mode 100644 index 0000000..6c241aa --- /dev/null +++ b/conf/druid/auto/historical/runtime.properties @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N + +# Processing threads and buffers +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine diff --git a/conf/druid/auto/indexer/runtime.properties b/conf/druid/auto/indexer/runtime.properties new file mode 100644 index 0000000..6d5f718 --- /dev/null +++ b/conf/druid/auto/indexer/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/indexer +druid.plaintextPort=8091 + +# Number of tasks (druid.worker.capacity) is automatically +# determined based on available processor. + +# Task launch parameters +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# Processing threads and buffers on Indexer +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/auto/middleManager/runtime.properties b/conf/druid/auto/middleManager/runtime.properties new file mode 100644 index 0000000..28d189b --- /dev/null +++ b/conf/druid/auto/middleManager/runtime.properties @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks (druid.worker.capacity) and memory usage per task (druid.indexer.runner.javaOptsArray) is automatically +# determined based on available memory. For details on how to manually set parameters, see: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#middlemanager + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# Processing threads and buffers on Peons +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/auto/router/runtime.properties b/conf/druid/auto/router/runtime.properties new file mode 100644 index 0000000..3858dec --- /dev/null +++ b/conf/druid/auto/router/runtime.properties @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/conf/druid/cluster/_common/common.runtime.properties b/conf/druid/cluster/_common/common.runtime.properties new file mode 100644 index 0000000..b0adb06 --- /dev/null +++ b/conf/druid/cluster/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/cluster/_common/log4j2.xml b/conf/druid/cluster/_common/log4j2.xml new file mode 100644 index 0000000..f55b700 --- /dev/null +++ b/conf/druid/cluster/_common/log4j2.xml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/druid/cluster/data/historical/jvm.config b/conf/druid/cluster/data/historical/jvm.config new file mode 100644 index 0000000..891312f --- /dev/null +++ b/conf/druid/cluster/data/historical/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms8g +-Xmx8g +-XX:MaxDirectMemorySize=13g +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/cluster/data/historical/main.config b/conf/druid/cluster/data/historical/main.config new file mode 100644 index 0000000..42be713 --- /dev/null +++ b/conf/druid/cluster/data/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/conf/druid/cluster/data/historical/runtime.properties b/conf/druid/cluster/data/historical/runtime.properties new file mode 100644 index 0000000..a7a640b --- /dev/null +++ b/conf/druid/cluster/data/historical/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=4 +druid.processing.numThreads=15 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=256MiB diff --git a/conf/druid/cluster/data/indexer/jvm.config b/conf/druid/cluster/data/indexer/jvm.config new file mode 100644 index 0000000..4611a65 --- /dev/null +++ b/conf/druid/cluster/data/indexer/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=4g +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/cluster/data/indexer/main.config b/conf/druid/cluster/data/indexer/main.config new file mode 100644 index 0000000..5183399 --- /dev/null +++ b/conf/druid/cluster/data/indexer/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server indexer diff --git a/conf/druid/cluster/data/indexer/runtime.properties b/conf/druid/cluster/data/indexer/runtime.properties new file mode 100644 index 0000000..7c6018c --- /dev/null +++ b/conf/druid/cluster/data/indexer/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/indexer +druid.plaintextPort=8091 + +# Number of tasks per indexer +druid.worker.capacity=4 + +# Task launch parameters +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Indexer +druid.processing.numMergeBuffers=2 +druid.processing.buffer.sizeBytes=100MiB +druid.processing.numThreads=4 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/cluster/data/middleManager/jvm.config b/conf/druid/cluster/data/middleManager/jvm.config new file mode 100644 index 0000000..c145532 --- /dev/null +++ b/conf/druid/cluster/data/middleManager/jvm.config @@ -0,0 +1,8 @@ +-server +-Xms128m +-Xmx128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/cluster/data/middleManager/main.config b/conf/druid/cluster/data/middleManager/main.config new file mode 100644 index 0000000..0e62733 --- /dev/null +++ b/conf/druid/cluster/data/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/conf/druid/cluster/data/middleManager/runtime.properties b/conf/druid/cluster/data/middleManager/runtime.properties new file mode 100644 index 0000000..55570b6 --- /dev/null +++ b/conf/druid/cluster/data/middleManager/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=4 +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/cluster/master/coordinator-overlord/jvm.config b/conf/druid/cluster/master/coordinator-overlord/jvm.config new file mode 100644 index 0000000..5df7d60 --- /dev/null +++ b/conf/druid/cluster/master/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms15g +-Xmx15g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/conf/druid/cluster/master/coordinator-overlord/main.config b/conf/druid/cluster/master/coordinator-overlord/main.config new file mode 100644 index 0000000..a74101f --- /dev/null +++ b/conf/druid/cluster/master/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/conf/druid/cluster/master/coordinator-overlord/runtime.properties b/conf/druid/cluster/master/coordinator-overlord/runtime.properties new file mode 100644 index 0000000..c053823 --- /dev/null +++ b/conf/druid/cluster/master/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/conf/druid/cluster/query/broker/jvm.config b/conf/druid/cluster/query/broker/jvm.config new file mode 100644 index 0000000..442a7b2 --- /dev/null +++ b/conf/druid/cluster/query/broker/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms12g +-Xmx12g +-XX:MaxDirectMemorySize=6g +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/cluster/query/broker/main.config b/conf/druid/cluster/query/broker/main.config new file mode 100644 index 0000000..9bc9a6f --- /dev/null +++ b/conf/druid/cluster/query/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/conf/druid/cluster/query/broker/runtime.properties b/conf/druid/cluster/query/broker/runtime.properties new file mode 100644 index 0000000..2b9a90d --- /dev/null +++ b/conf/druid/cluster/query/broker/runtime.properties @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=10MiB + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=6 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/conf/druid/cluster/query/router/jvm.config b/conf/druid/cluster/query/router/jvm.config new file mode 100644 index 0000000..a844c40 --- /dev/null +++ b/conf/druid/cluster/query/router/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms1g +-Xmx1g +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/cluster/query/router/main.config b/conf/druid/cluster/query/router/main.config new file mode 100644 index 0000000..4194b69 --- /dev/null +++ b/conf/druid/cluster/query/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/conf/druid/cluster/query/router/runtime.properties b/conf/druid/cluster/query/router/runtime.properties new file mode 100644 index 0000000..497d3b4 --- /dev/null +++ b/conf/druid/cluster/query/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/conf/druid/single-server/.DS_Store b/conf/druid/single-server/.DS_Store deleted file mode 100644 index 30e48e96ef51b00557c73e2f161974aa7f586923..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyG{c!5S)b+ktj$>=^vor4^C0|0)7BU8brhilJr;QyYgw7eKe~Fjn9;mkY1|&-6d~|1n7`DIf)|N&%a#URFy!scP%wab9a1{gv)HC%PNwLE#YP km>A`l3opkHk(7DO=iKiKhs2;W9(1C923!}J6!>oi&Z|xrUH||9 diff --git a/conf/druid/single-server/large/_common/common.runtime.properties b/conf/druid/single-server/large/_common/common.runtime.properties new file mode 100644 index 0000000..b0adb06 --- /dev/null +++ b/conf/druid/single-server/large/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/single-server/large/_common/log4j2.xml b/conf/druid/single-server/large/_common/log4j2.xml new file mode 100644 index 0000000..66dc13d --- /dev/null +++ b/conf/druid/single-server/large/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/druid/single-server/large/broker/jvm.config b/conf/druid/single-server/large/broker/jvm.config new file mode 100644 index 0000000..6c43c24 --- /dev/null +++ b/conf/druid/single-server/large/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms12g +-Xmx12g +-XX:MaxDirectMemorySize=11g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/large/broker/main.config b/conf/druid/single-server/large/broker/main.config new file mode 100644 index 0000000..9bc9a6f --- /dev/null +++ b/conf/druid/single-server/large/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/conf/druid/single-server/large/broker/runtime.properties b/conf/druid/single-server/large/broker/runtime.properties new file mode 100644 index 0000000..03d7686 --- /dev/null +++ b/conf/druid/single-server/large/broker/runtime.properties @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=10MiB + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=16 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/conf/druid/single-server/large/coordinator-overlord/jvm.config b/conf/druid/single-server/large/coordinator-overlord/jvm.config new file mode 100644 index 0000000..5df7d60 --- /dev/null +++ b/conf/druid/single-server/large/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms15g +-Xmx15g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/conf/druid/single-server/large/coordinator-overlord/main.config b/conf/druid/single-server/large/coordinator-overlord/main.config new file mode 100644 index 0000000..a74101f --- /dev/null +++ b/conf/druid/single-server/large/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/conf/druid/single-server/large/coordinator-overlord/runtime.properties b/conf/druid/single-server/large/coordinator-overlord/runtime.properties new file mode 100644 index 0000000..c053823 --- /dev/null +++ b/conf/druid/single-server/large/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/conf/druid/single-server/large/historical/jvm.config b/conf/druid/single-server/large/historical/jvm.config new file mode 100644 index 0000000..16e1f5d --- /dev/null +++ b/conf/druid/single-server/large/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms16g +-Xmx16g +-XX:MaxDirectMemorySize=25g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/large/historical/main.config b/conf/druid/single-server/large/historical/main.config new file mode 100644 index 0000000..42be713 --- /dev/null +++ b/conf/druid/single-server/large/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/conf/druid/single-server/large/historical/runtime.properties b/conf/druid/single-server/large/historical/runtime.properties new file mode 100644 index 0000000..d88eb1d --- /dev/null +++ b/conf/druid/single-server/large/historical/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=8 +druid.processing.numThreads=31 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=512MiB diff --git a/conf/druid/single-server/large/middleManager/jvm.config b/conf/druid/single-server/large/middleManager/jvm.config new file mode 100644 index 0000000..82597e0 --- /dev/null +++ b/conf/druid/single-server/large/middleManager/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms256m +-Xmx256m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/large/middleManager/main.config b/conf/druid/single-server/large/middleManager/main.config new file mode 100644 index 0000000..0e62733 --- /dev/null +++ b/conf/druid/single-server/large/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/conf/druid/single-server/large/middleManager/runtime.properties b/conf/druid/single-server/large/middleManager/runtime.properties new file mode 100644 index 0000000..62a8d1c --- /dev/null +++ b/conf/druid/single-server/large/middleManager/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=8 +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/single-server/large/router/jvm.config b/conf/druid/single-server/large/router/jvm.config new file mode 100644 index 0000000..a844c40 --- /dev/null +++ b/conf/druid/single-server/large/router/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms1g +-Xmx1g +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/large/router/main.config b/conf/druid/single-server/large/router/main.config new file mode 100644 index 0000000..4194b69 --- /dev/null +++ b/conf/druid/single-server/large/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/conf/druid/single-server/large/router/runtime.properties b/conf/druid/single-server/large/router/runtime.properties new file mode 100644 index 0000000..497d3b4 --- /dev/null +++ b/conf/druid/single-server/large/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/conf/druid/single-server/medium/_common/common.runtime.properties b/conf/druid/single-server/medium/_common/common.runtime.properties new file mode 100644 index 0000000..b0adb06 --- /dev/null +++ b/conf/druid/single-server/medium/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/single-server/medium/_common/log4j2.xml b/conf/druid/single-server/medium/_common/log4j2.xml new file mode 100644 index 0000000..66dc13d --- /dev/null +++ b/conf/druid/single-server/medium/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/druid/single-server/medium/broker/jvm.config b/conf/druid/single-server/medium/broker/jvm.config new file mode 100644 index 0000000..a4bf3d9 --- /dev/null +++ b/conf/druid/single-server/medium/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms8g +-Xmx8g +-XX:MaxDirectMemorySize=5g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/medium/broker/main.config b/conf/druid/single-server/medium/broker/main.config new file mode 100644 index 0000000..9bc9a6f --- /dev/null +++ b/conf/druid/single-server/medium/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/conf/druid/single-server/medium/broker/runtime.properties b/conf/druid/single-server/medium/broker/runtime.properties new file mode 100644 index 0000000..d46d6de --- /dev/null +++ b/conf/druid/single-server/medium/broker/runtime.properties @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=10MiB + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=4 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/conf/druid/single-server/medium/coordinator-overlord/jvm.config b/conf/druid/single-server/medium/coordinator-overlord/jvm.config new file mode 100644 index 0000000..dbddd50 --- /dev/null +++ b/conf/druid/single-server/medium/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms9g +-Xmx9g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/conf/druid/single-server/medium/coordinator-overlord/main.config b/conf/druid/single-server/medium/coordinator-overlord/main.config new file mode 100644 index 0000000..a74101f --- /dev/null +++ b/conf/druid/single-server/medium/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/conf/druid/single-server/medium/coordinator-overlord/runtime.properties b/conf/druid/single-server/medium/coordinator-overlord/runtime.properties new file mode 100644 index 0000000..c053823 --- /dev/null +++ b/conf/druid/single-server/medium/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/conf/druid/single-server/medium/historical/jvm.config b/conf/druid/single-server/medium/historical/jvm.config new file mode 100644 index 0000000..f31ae92 --- /dev/null +++ b/conf/druid/single-server/medium/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms8g +-Xmx8g +-XX:MaxDirectMemorySize=13g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/medium/historical/main.config b/conf/druid/single-server/medium/historical/main.config new file mode 100644 index 0000000..42be713 --- /dev/null +++ b/conf/druid/single-server/medium/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/conf/druid/single-server/medium/historical/runtime.properties b/conf/druid/single-server/medium/historical/runtime.properties new file mode 100644 index 0000000..a7a640b --- /dev/null +++ b/conf/druid/single-server/medium/historical/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=4 +druid.processing.numThreads=15 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=256MiB diff --git a/conf/druid/single-server/medium/middleManager/jvm.config b/conf/druid/single-server/medium/middleManager/jvm.config new file mode 100644 index 0000000..82597e0 --- /dev/null +++ b/conf/druid/single-server/medium/middleManager/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms256m +-Xmx256m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/medium/middleManager/main.config b/conf/druid/single-server/medium/middleManager/main.config new file mode 100644 index 0000000..0e62733 --- /dev/null +++ b/conf/druid/single-server/medium/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/conf/druid/single-server/medium/middleManager/runtime.properties b/conf/druid/single-server/medium/middleManager/runtime.properties new file mode 100644 index 0000000..55570b6 --- /dev/null +++ b/conf/druid/single-server/medium/middleManager/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=4 +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/single-server/medium/router/jvm.config b/conf/druid/single-server/medium/router/jvm.config new file mode 100644 index 0000000..6fc9b4c --- /dev/null +++ b/conf/druid/single-server/medium/router/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms512m +-Xmx512m +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/medium/router/main.config b/conf/druid/single-server/medium/router/main.config new file mode 100644 index 0000000..4194b69 --- /dev/null +++ b/conf/druid/single-server/medium/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/conf/druid/single-server/medium/router/runtime.properties b/conf/druid/single-server/medium/router/runtime.properties new file mode 100644 index 0000000..497d3b4 --- /dev/null +++ b/conf/druid/single-server/medium/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties b/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties index 374e77c..a841ff7 100644 --- a/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties +++ b/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties @@ -24,7 +24,7 @@ # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. # More info: https://druid.apache.org/docs/latest/operations/including-extensions.html -druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "postgresql-metadata-storage", "druid-s3-extensions"] +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "postgresql-metadata-storage", "druid-s3-extensions"] # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory # and uncomment the line below to point to your directory. @@ -77,17 +77,13 @@ druid.metadata.storage.connector.port=1527 # # For local disk (only viable in a cluster if this is a network mount): -#druid.storage.type=local -#druid.storage.storageDirectory=var/druid/segments +# druid.storage.type=local +# druid.storage.storageDirectory=var/druid/segments # For HDFS: #druid.storage.type=hdfs #druid.storage.storageDirectory=/druid/segments - -druid.s3.endpoint.url=http://localhost:9000 -druid.s3.enablePathStyleAccess=true - # For S3: druid.storage.type=s3 druid.storage.bucket=storage-bucket @@ -95,13 +91,16 @@ druid.storage.baseKey=druid/segments druid.s3.accessKey=access_key druid.s3.secretKey=secret_key +druid.s3.endpoint.url=http://localhost:9000 +druid.s3.enablePathStyleAccess=true + # # Indexing service logs # # For local disk (only viable in a cluster if this is a network mount): -#druid.indexer.logs.type=file -#druid.indexer.logs.directory=var/druid/indexing-logs +# druid.indexer.logs.type=file +# druid.indexer.logs.directory=var/druid/indexing-logs # For HDFS: #druid.indexer.logs.type=hdfs @@ -120,7 +119,7 @@ druid.selectors.indexing.serviceName=druid/overlord druid.selectors.coordinator.serviceName=druid/coordinator # -# Monitoringf +# Monitoring # druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] @@ -135,7 +134,7 @@ druid.indexing.doubleStorage=double # # Security # -druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] # @@ -143,7 +142,20 @@ druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid. # druid.sql.enable=true +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + # # Lookups # druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/single-server/micro-quickstart/_common/log4j2.xml b/conf/druid/single-server/micro-quickstart/_common/log4j2.xml index 2f88d04..66dc13d 100644 --- a/conf/druid/single-server/micro-quickstart/_common/log4j2.xml +++ b/conf/druid/single-server/micro-quickstart/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/conf/druid/single-server/micro-quickstart/broker/runtime.properties b/conf/druid/single-server/micro-quickstart/broker/runtime.properties index 5cb791c..92b85f8 100644 --- a/conf/druid/single-server/micro-quickstart/broker/runtime.properties +++ b/conf/druid/single-server/micro-quickstart/broker/runtime.properties @@ -25,16 +25,13 @@ druid.server.http.numThreads=12 # HTTP client settings druid.broker.http.numConnections=10 -druid.broker.http.maxQueuedBytes=5000000 +druid.broker.http.maxQueuedBytes=5MiB # Processing threads and buffers -druid.processing.buffer.sizeBytes=100000000 +druid.processing.buffer.sizeBytes=100MiB druid.processing.numMergeBuffers=2 -druid.processing.numThreads=1 druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead druid.broker.cache.useCache=false druid.broker.cache.populateCache=false - -druid.sql.planner.metadataRefreshPeriod=PT1S diff --git a/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties b/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties index 15f7254..c053823 100644 --- a/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties +++ b/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties @@ -21,17 +21,13 @@ druid.service=druid/coordinator druid.plaintextPort=8081 druid.coordinator.startDelay=PT10S -druid.coordinator.period=PT1S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S # Run the overlord service in the coordinator process druid.coordinator.asOverlord.enabled=true druid.coordinator.asOverlord.overlordService=druid/overlord -druid.indexer.queue.startDelay=PT1S +druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote druid.indexer.storage.type=metadata - -druid.manager.segments.pollDuration=PT1S -druid.manager.config.pollDuration=PT1S -druid.manager.rules.pollDuration=PT1S \ No newline at end of file diff --git a/conf/druid/single-server/micro-quickstart/historical/runtime.properties b/conf/druid/single-server/micro-quickstart/historical/runtime.properties index 594abaf..76eaa5d 100644 --- a/conf/druid/single-server/micro-quickstart/historical/runtime.properties +++ b/conf/druid/single-server/micro-quickstart/historical/runtime.properties @@ -24,17 +24,16 @@ druid.plaintextPort=8083 druid.server.http.numThreads=12 # Processing threads and buffers -druid.processing.buffer.sizeBytes=200000000 +druid.processing.buffer.sizeBytes=200MiB druid.processing.numMergeBuffers=2 druid.processing.numThreads=2 druid.processing.tmpDir=var/druid/processing # Segment storage -druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] -druid.server.maxSize=300000000000 +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] # Query cache druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine -druid.cache.sizeInBytes=10000000 +druid.cache.sizeInBytes=10MiB diff --git a/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties b/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties index 31e10cc..091e118 100644 --- a/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties +++ b/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties @@ -21,18 +21,19 @@ druid.service=druid/middleManager druid.plaintextPort=8091 # Number of tasks per middleManager -druid.worker.capacity=4 +druid.worker.capacity=2 +druid.worker.baseTaskDirs=[\"var/druid/task\"] # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -druid.indexer.task.baseTaskDir=var/druid/task +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] # HTTP server threads druid.server.http.numThreads=12 # Processing threads and buffers on Peons druid.indexer.fork.property.druid.processing.numMergeBuffers=2 -druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100000000 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB druid.indexer.fork.property.druid.processing.numThreads=1 # Hadoop indexing diff --git a/conf/druid/single-server/micro-quickstart/router/runtime.properties b/conf/druid/single-server/micro-quickstart/router/runtime.properties index 2131464..497d3b4 100644 --- a/conf/druid/single-server/micro-quickstart/router/runtime.properties +++ b/conf/druid/single-server/micro-quickstart/router/runtime.properties @@ -20,19 +20,11 @@ druid.service=druid/router druid.plaintextPort=8888 -druid.global.http.numMaxThreads=16 - -# Broker -druid.borker.http.numMaxThreads=16 - # HTTP proxy druid.router.http.numConnections=50 druid.router.http.readTimeout=PT5M -druid.router.http.numMaxThreads=16 -druid.server.http.numThreads=16 - -# Indexer -druid.server.http.numThreads=16 +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 # Service discovery druid.router.defaultBrokerServiceName=druid/broker @@ -40,6 +32,3 @@ druid.router.coordinatorServiceName=druid/coordinator # Management proxy to coordinator / overlord: required for unified web console. druid.router.managementProxy.enabled=true - -# Processing -druid.processing.numThreads=4 \ No newline at end of file diff --git a/conf/druid/single-server/nano-quickstart/.DS_Store b/conf/druid/single-server/nano-quickstart/.DS_Store deleted file mode 100644 index 8a3223a90f57b157368fe28c3261d17a94cf7592..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKISv9b4733WBpOP}e1RWC2wuPgKtKs>f#|p5T|ABPQGgaYC_rP$nZ)rV$`tFh zi0J&XpNULFWCS;qs|{VVee;g>GNM2@&NxaVr~P?%*fyQ)*8$@WWi2}}kB|K3U|TdQ zKn17(6`%rC;KK@JfgO!Md@hfp0#xAl6|nC^fg9GuHqbvE7`z1l4iI+3+i0p zwti-h{ zrvHB?aYY5Fz+Wk#gT-nw$CI+Qb{=Q7w!k-V%eleLFn0c7o_bQ`6`Nzf VCbof2N8IT^{tTEdG%E0I1@1F}6_Ee{ diff --git a/conf/druid/single-server/nano-quickstart/.idea/.gitignore b/conf/druid/single-server/nano-quickstart/.idea/.gitignore deleted file mode 100644 index 73f69e0..0000000 --- a/conf/druid/single-server/nano-quickstart/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/Project_Default.xml b/conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 1af6197..0000000 --- a/conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - \ No newline at end of file diff --git a/conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/profiles_settings.xml b/conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/conf/druid/single-server/nano-quickstart/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/conf/druid/single-server/nano-quickstart/.idea/misc.xml b/conf/druid/single-server/nano-quickstart/.idea/misc.xml deleted file mode 100644 index 5be1be9..0000000 --- a/conf/druid/single-server/nano-quickstart/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/conf/druid/single-server/nano-quickstart/.idea/modules.xml b/conf/druid/single-server/nano-quickstart/.idea/modules.xml deleted file mode 100644 index 52ce35a..0000000 --- a/conf/druid/single-server/nano-quickstart/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/conf/druid/single-server/nano-quickstart/.idea/nano-quickstart.iml b/conf/druid/single-server/nano-quickstart/.idea/nano-quickstart.iml deleted file mode 100644 index d0876a7..0000000 --- a/conf/druid/single-server/nano-quickstart/.idea/nano-quickstart.iml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties b/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties index 374e77c..a841ff7 100644 --- a/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties +++ b/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties @@ -24,7 +24,7 @@ # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. # More info: https://druid.apache.org/docs/latest/operations/including-extensions.html -druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "postgresql-metadata-storage", "druid-s3-extensions"] +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "postgresql-metadata-storage", "druid-s3-extensions"] # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory # and uncomment the line below to point to your directory. @@ -77,17 +77,13 @@ druid.metadata.storage.connector.port=1527 # # For local disk (only viable in a cluster if this is a network mount): -#druid.storage.type=local -#druid.storage.storageDirectory=var/druid/segments +# druid.storage.type=local +# druid.storage.storageDirectory=var/druid/segments # For HDFS: #druid.storage.type=hdfs #druid.storage.storageDirectory=/druid/segments - -druid.s3.endpoint.url=http://localhost:9000 -druid.s3.enablePathStyleAccess=true - # For S3: druid.storage.type=s3 druid.storage.bucket=storage-bucket @@ -95,13 +91,16 @@ druid.storage.baseKey=druid/segments druid.s3.accessKey=access_key druid.s3.secretKey=secret_key +druid.s3.endpoint.url=http://localhost:9000 +druid.s3.enablePathStyleAccess=true + # # Indexing service logs # # For local disk (only viable in a cluster if this is a network mount): -#druid.indexer.logs.type=file -#druid.indexer.logs.directory=var/druid/indexing-logs +# druid.indexer.logs.type=file +# druid.indexer.logs.directory=var/druid/indexing-logs # For HDFS: #druid.indexer.logs.type=hdfs @@ -120,7 +119,7 @@ druid.selectors.indexing.serviceName=druid/overlord druid.selectors.coordinator.serviceName=druid/coordinator # -# Monitoringf +# Monitoring # druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] @@ -135,7 +134,7 @@ druid.indexing.doubleStorage=double # # Security # -druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] # @@ -143,7 +142,20 @@ druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid. # druid.sql.enable=true +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + # # Lookups # druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/single-server/nano-quickstart/_common/log4j2.xml b/conf/druid/single-server/nano-quickstart/_common/log4j2.xml index 658f990..66dc13d 100644 --- a/conf/druid/single-server/nano-quickstart/_common/log4j2.xml +++ b/conf/druid/single-server/nano-quickstart/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - - + + + + + - + - + - + - + - + - + diff --git a/conf/druid/single-server/nano-quickstart/broker/runtime.properties b/conf/druid/single-server/nano-quickstart/broker/runtime.properties index 657f374..b255651 100644 --- a/conf/druid/single-server/nano-quickstart/broker/runtime.properties +++ b/conf/druid/single-server/nano-quickstart/broker/runtime.properties @@ -25,16 +25,13 @@ druid.server.http.numThreads=6 # HTTP client settings druid.broker.http.numConnections=5 -druid.broker.http.maxQueuedBytes=5000000 +druid.broker.http.maxQueuedBytes=5MiB # Processing threads and buffers -druid.processing.buffer.sizeBytes=50000000 +druid.processing.buffer.sizeBytes=50MiB druid.processing.numMergeBuffers=2 -druid.processing.numThreads=1 druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead druid.broker.cache.useCache=false druid.broker.cache.populateCache=false - -druid.sql.planner.metadataRefreshPeriod=PT1S diff --git a/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties b/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties index 15f7254..c053823 100644 --- a/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties +++ b/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties @@ -21,17 +21,13 @@ druid.service=druid/coordinator druid.plaintextPort=8081 druid.coordinator.startDelay=PT10S -druid.coordinator.period=PT1S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S # Run the overlord service in the coordinator process druid.coordinator.asOverlord.enabled=true druid.coordinator.asOverlord.overlordService=druid/overlord -druid.indexer.queue.startDelay=PT1S +druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote druid.indexer.storage.type=metadata - -druid.manager.segments.pollDuration=PT1S -druid.manager.config.pollDuration=PT1S -druid.manager.rules.pollDuration=PT1S \ No newline at end of file diff --git a/conf/druid/single-server/nano-quickstart/historical/runtime.properties b/conf/druid/single-server/nano-quickstart/historical/runtime.properties index f2e8d3b..e2720fb 100644 --- a/conf/druid/single-server/nano-quickstart/historical/runtime.properties +++ b/conf/druid/single-server/nano-quickstart/historical/runtime.properties @@ -24,17 +24,16 @@ druid.plaintextPort=8083 druid.server.http.numThreads=6 # Processing threads and buffers -druid.processing.buffer.sizeBytes=50000000 +druid.processing.buffer.sizeBytes=50MiB druid.processing.numMergeBuffers=2 druid.processing.numThreads=1 druid.processing.tmpDir=var/druid/processing # Segment storage -druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] -druid.server.maxSize=300000000000 +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] # Query cache druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine -druid.cache.sizeInBytes=50000000 +druid.cache.sizeInBytes=50MiB diff --git a/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties b/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties index c0e42aa..1dde859 100644 --- a/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties +++ b/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties @@ -21,18 +21,19 @@ druid.service=druid/middleManager druid.plaintextPort=8091 # Number of tasks per middleManager -druid.worker.capacity=4 +druid.worker.capacity=2 +druid.worker.baseTaskDirs=[\"var/druid/task\"] # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -druid.indexer.task.baseTaskDir=var/druid/task +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms256m","-Xmx256m","-XX:MaxDirectMemorySize=300m","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] # HTTP server threads druid.server.http.numThreads=6 # Processing threads and buffers on Peons druid.indexer.fork.property.druid.processing.numMergeBuffers=2 -druid.indexer.fork.property.druid.processing.buffer.sizeBytes=25000000 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=25MiB druid.indexer.fork.property.druid.processing.numThreads=1 # Hadoop indexing diff --git a/conf/druid/single-server/nano-quickstart/router/runtime.properties b/conf/druid/single-server/nano-quickstart/router/runtime.properties index d1103ef..1868e9b 100644 --- a/conf/druid/single-server/nano-quickstart/router/runtime.properties +++ b/conf/druid/single-server/nano-quickstart/router/runtime.properties @@ -20,16 +20,11 @@ druid.service=druid/router druid.plaintextPort=8888 -druid.global.http.numMaxThreads=100 - -# Broker -druid.broker.http.numMaxThreads=100 - # HTTP proxy druid.router.http.numConnections=25 druid.router.http.readTimeout=PT5M -druid.router.http.numMaxThreads=100 -druid.server.http.numThreads=4 +druid.router.http.numMaxThreads=50 +druid.server.http.numThreads=50 # Service discovery druid.router.defaultBrokerServiceName=druid/broker @@ -37,6 +32,3 @@ druid.router.coordinatorServiceName=druid/coordinator # Management proxy to coordinator / overlord: required for unified web console. druid.router.managementProxy.enabled=true - -# Processing -druid.processing.numThreads=1 \ No newline at end of file diff --git a/conf/druid/single-server/small/_common/common.runtime.properties b/conf/druid/single-server/small/_common/common.runtime.properties new file mode 100644 index 0000000..b0adb06 --- /dev/null +++ b/conf/druid/single-server/small/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/single-server/small/_common/log4j2.xml b/conf/druid/single-server/small/_common/log4j2.xml new file mode 100644 index 0000000..66dc13d --- /dev/null +++ b/conf/druid/single-server/small/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/druid/single-server/small/broker/jvm.config b/conf/druid/single-server/small/broker/jvm.config new file mode 100644 index 0000000..1017bce --- /dev/null +++ b/conf/druid/single-server/small/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=3g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/small/broker/main.config b/conf/druid/single-server/small/broker/main.config new file mode 100644 index 0000000..9bc9a6f --- /dev/null +++ b/conf/druid/single-server/small/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/conf/druid/single-server/small/broker/runtime.properties b/conf/druid/single-server/small/broker/runtime.properties new file mode 100644 index 0000000..dbe913a --- /dev/null +++ b/conf/druid/single-server/small/broker/runtime.properties @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=50 + +# HTTP client settings +druid.broker.http.numConnections=40 +druid.broker.http.maxQueuedBytes=5MiB + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=2 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/conf/druid/single-server/small/coordinator-overlord/jvm.config b/conf/druid/single-server/small/coordinator-overlord/jvm.config new file mode 100644 index 0000000..3417668 --- /dev/null +++ b/conf/druid/single-server/small/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms4500m +-Xmx4500m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/conf/druid/single-server/small/coordinator-overlord/main.config b/conf/druid/single-server/small/coordinator-overlord/main.config new file mode 100644 index 0000000..a74101f --- /dev/null +++ b/conf/druid/single-server/small/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/conf/druid/single-server/small/coordinator-overlord/runtime.properties b/conf/druid/single-server/small/coordinator-overlord/runtime.properties new file mode 100644 index 0000000..c053823 --- /dev/null +++ b/conf/druid/single-server/small/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/conf/druid/single-server/small/historical/jvm.config b/conf/druid/single-server/small/historical/jvm.config new file mode 100644 index 0000000..2190607 --- /dev/null +++ b/conf/druid/single-server/small/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=8g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/small/historical/main.config b/conf/druid/single-server/small/historical/main.config new file mode 100644 index 0000000..42be713 --- /dev/null +++ b/conf/druid/single-server/small/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/conf/druid/single-server/small/historical/runtime.properties b/conf/druid/single-server/small/historical/runtime.properties new file mode 100644 index 0000000..6903c28 --- /dev/null +++ b/conf/druid/single-server/small/historical/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=50 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=2 +druid.processing.numThreads=7 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=256MiB diff --git a/conf/druid/single-server/small/middleManager/jvm.config b/conf/druid/single-server/small/middleManager/jvm.config new file mode 100644 index 0000000..a586218 --- /dev/null +++ b/conf/druid/single-server/small/middleManager/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms128m +-Xmx128m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/small/middleManager/main.config b/conf/druid/single-server/small/middleManager/main.config new file mode 100644 index 0000000..0e62733 --- /dev/null +++ b/conf/druid/single-server/small/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/conf/druid/single-server/small/middleManager/runtime.properties b/conf/druid/single-server/small/middleManager/runtime.properties new file mode 100644 index 0000000..846294f --- /dev/null +++ b/conf/druid/single-server/small/middleManager/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=3 +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] + +# HTTP server threads +druid.server.http.numThreads=50 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/single-server/small/router/jvm.config b/conf/druid/single-server/small/router/jvm.config new file mode 100644 index 0000000..ad8a52d --- /dev/null +++ b/conf/druid/single-server/small/router/jvm.config @@ -0,0 +1,11 @@ +-server +-Xms512m +-Xmx512m +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/small/router/main.config b/conf/druid/single-server/small/router/main.config new file mode 100644 index 0000000..4194b69 --- /dev/null +++ b/conf/druid/single-server/small/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/conf/druid/single-server/small/router/runtime.properties b/conf/druid/single-server/small/router/runtime.properties new file mode 100644 index 0000000..497d3b4 --- /dev/null +++ b/conf/druid/single-server/small/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/conf/druid/single-server/xlarge/_common/common.runtime.properties b/conf/druid/single-server/xlarge/_common/common.runtime.properties new file mode 100644 index 0000000..b0adb06 --- /dev/null +++ b/conf/druid/single-server/xlarge/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/conf/druid/single-server/xlarge/_common/log4j2.xml b/conf/druid/single-server/xlarge/_common/log4j2.xml new file mode 100644 index 0000000..66dc13d --- /dev/null +++ b/conf/druid/single-server/xlarge/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/druid/single-server/xlarge/broker/jvm.config b/conf/druid/single-server/xlarge/broker/jvm.config new file mode 100644 index 0000000..f83ad0e --- /dev/null +++ b/conf/druid/single-server/xlarge/broker/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms16g +-Xmx16g +-XX:MaxDirectMemorySize=12g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/xlarge/broker/main.config b/conf/druid/single-server/xlarge/broker/main.config new file mode 100644 index 0000000..9bc9a6f --- /dev/null +++ b/conf/druid/single-server/xlarge/broker/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server broker diff --git a/conf/druid/single-server/xlarge/broker/runtime.properties b/conf/druid/single-server/xlarge/broker/runtime.properties new file mode 100644 index 0000000..03d7686 --- /dev/null +++ b/conf/druid/single-server/xlarge/broker/runtime.properties @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=10MiB + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=16 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/conf/druid/single-server/xlarge/coordinator-overlord/jvm.config b/conf/druid/single-server/xlarge/coordinator-overlord/jvm.config new file mode 100644 index 0000000..f3ca0fd --- /dev/null +++ b/conf/druid/single-server/xlarge/coordinator-overlord/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms18g +-Xmx18g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log diff --git a/conf/druid/single-server/xlarge/coordinator-overlord/main.config b/conf/druid/single-server/xlarge/coordinator-overlord/main.config new file mode 100644 index 0000000..a74101f --- /dev/null +++ b/conf/druid/single-server/xlarge/coordinator-overlord/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server coordinator diff --git a/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties b/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties new file mode 100644 index 0000000..c053823 --- /dev/null +++ b/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/conf/druid/single-server/xlarge/historical/jvm.config b/conf/druid/single-server/xlarge/historical/jvm.config new file mode 100644 index 0000000..92553d3 --- /dev/null +++ b/conf/druid/single-server/xlarge/historical/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms24g +-Xmx24g +-XX:MaxDirectMemorySize=44g +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/xlarge/historical/main.config b/conf/druid/single-server/xlarge/historical/main.config new file mode 100644 index 0000000..42be713 --- /dev/null +++ b/conf/druid/single-server/xlarge/historical/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server historical diff --git a/conf/druid/single-server/xlarge/historical/runtime.properties b/conf/druid/single-server/xlarge/historical/runtime.properties new file mode 100644 index 0000000..d19c09e --- /dev/null +++ b/conf/druid/single-server/xlarge/historical/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=16 +druid.processing.numThreads=63 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=2GiB diff --git a/conf/druid/single-server/xlarge/middleManager/jvm.config b/conf/druid/single-server/xlarge/middleManager/jvm.config new file mode 100644 index 0000000..82597e0 --- /dev/null +++ b/conf/druid/single-server/xlarge/middleManager/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms256m +-Xmx256m +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/xlarge/middleManager/main.config b/conf/druid/single-server/xlarge/middleManager/main.config new file mode 100644 index 0000000..0e62733 --- /dev/null +++ b/conf/druid/single-server/xlarge/middleManager/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server middleManager diff --git a/conf/druid/single-server/xlarge/middleManager/runtime.properties b/conf/druid/single-server/xlarge/middleManager/runtime.properties new file mode 100644 index 0000000..e1d848b --- /dev/null +++ b/conf/druid/single-server/xlarge/middleManager/runtime.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=16 +druid.worker.baseTaskDirs=[\"var/druid/task\"] + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/conf/druid/single-server/xlarge/router/jvm.config b/conf/druid/single-server/xlarge/router/jvm.config new file mode 100644 index 0000000..a844c40 --- /dev/null +++ b/conf/druid/single-server/xlarge/router/jvm.config @@ -0,0 +1,10 @@ +-server +-Xms1g +-Xmx1g +-XX:+UseG1GC +-XX:MaxDirectMemorySize=128m +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/conf/druid/single-server/xlarge/router/main.config b/conf/druid/single-server/xlarge/router/main.config new file mode 100644 index 0000000..4194b69 --- /dev/null +++ b/conf/druid/single-server/xlarge/router/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server router diff --git a/conf/druid/single-server/xlarge/router/runtime.properties b/conf/druid/single-server/xlarge/router/runtime.properties new file mode 100644 index 0000000..497d3b4 --- /dev/null +++ b/conf/druid/single-server/xlarge/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/conf/supervise/cluster/data.conf b/conf/supervise/cluster/data.conf new file mode 100644 index 0000000..f2a16e6 --- /dev/null +++ b/conf/supervise/cluster/data.conf @@ -0,0 +1,4 @@ +:verify bin/verify-java + +historical bin/run-druid historical conf/druid/cluster/data +middleManager bin/run-druid middleManager conf/druid/cluster/data diff --git a/conf/supervise/cluster/master-no-zk.conf b/conf/supervise/cluster/master-no-zk.conf new file mode 100644 index 0000000..2730387 --- /dev/null +++ b/conf/supervise/cluster/master-no-zk.conf @@ -0,0 +1,3 @@ +:verify bin/verify-java + +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/cluster/master diff --git a/conf/supervise/cluster/master-with-zk.conf b/conf/supervise/cluster/master-with-zk.conf new file mode 100644 index 0000000..2399827 --- /dev/null +++ b/conf/supervise/cluster/master-with-zk.conf @@ -0,0 +1,4 @@ +:verify bin/verify-java + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/cluster/master diff --git a/conf/supervise/cluster/query.conf b/conf/supervise/cluster/query.conf new file mode 100644 index 0000000..ead75fd --- /dev/null +++ b/conf/supervise/cluster/query.conf @@ -0,0 +1,4 @@ +:verify bin/verify-java + +broker bin/run-druid broker conf/druid/cluster/query +router bin/run-druid router conf/druid/cluster/query diff --git a/conf/supervise/single-server/large.conf b/conf/supervise/single-server/large.conf new file mode 100644 index 0000000..9dde5ea --- /dev/null +++ b/conf/supervise/single-server/large.conf @@ -0,0 +1,11 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:notify bin/greet +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/large +broker bin/run-druid broker conf/druid/single-server/large +router bin/run-druid router conf/druid/single-server/large +historical bin/run-druid historical conf/druid/single-server/large +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/large diff --git a/conf/supervise/single-server/medium.conf b/conf/supervise/single-server/medium.conf new file mode 100644 index 0000000..78d43ac --- /dev/null +++ b/conf/supervise/single-server/medium.conf @@ -0,0 +1,11 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:notify bin/greet +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/medium +broker bin/run-druid broker conf/druid/single-server/medium +router bin/run-druid router conf/druid/single-server/medium +historical bin/run-druid historical conf/druid/single-server/medium +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/medium diff --git a/conf/supervise/single-server/micro-quickstart.conf b/conf/supervise/single-server/micro-quickstart.conf index de1c0f8..78d43a3 100644 --- a/conf/supervise/single-server/micro-quickstart.conf +++ b/conf/supervise/single-server/micro-quickstart.conf @@ -1,5 +1,6 @@ :verify bin/verify-java :verify bin/verify-default-ports +:notify bin/greet :kill-timeout 10 !p10 zk bin/run-zk conf diff --git a/conf/supervise/single-server/nano-quickstart.conf b/conf/supervise/single-server/nano-quickstart.conf index 8baf769..918b368 100644 --- a/conf/supervise/single-server/nano-quickstart.conf +++ b/conf/supervise/single-server/nano-quickstart.conf @@ -1,5 +1,6 @@ :verify bin/verify-java :verify bin/verify-default-ports +:notify bin/greet :kill-timeout 10 !p10 zk bin/run-zk conf diff --git a/conf/supervise/single-server/small.conf b/conf/supervise/single-server/small.conf new file mode 100644 index 0000000..ef88416 --- /dev/null +++ b/conf/supervise/single-server/small.conf @@ -0,0 +1,11 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:notify bin/greet +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/small +broker bin/run-druid broker conf/druid/single-server/small +router bin/run-druid router conf/druid/single-server/small +historical bin/run-druid historical conf/druid/single-server/small +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/small diff --git a/conf/supervise/single-server/xlarge.conf b/conf/supervise/single-server/xlarge.conf new file mode 100644 index 0000000..e0d6c63 --- /dev/null +++ b/conf/supervise/single-server/xlarge.conf @@ -0,0 +1,11 @@ +:verify bin/verify-java +:verify bin/verify-default-ports +:notify bin/greet +:kill-timeout 10 + +!p10 zk bin/run-zk conf +coordinator-overlord bin/run-druid coordinator-overlord conf/druid/single-server/xlarge +broker bin/run-druid broker conf/druid/single-server/xlarge +router bin/run-druid router conf/druid/single-server/xlarge +historical bin/run-druid historical conf/druid/single-server/xlarge +!p90 middleManager bin/run-druid middleManager conf/druid/single-server/xlarge diff --git a/conf/zk/log4j2.xml b/conf/zk/log4j2.xml index 0ad3c40..ada12f3 100644 --- a/conf/zk/log4j2.xml +++ b/conf/zk/log4j2.xml @@ -19,14 +19,37 @@ --> - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/patches/conf/0001-chore-conf-extensions-postgresql-and-s3.patch b/patches/conf/0001-chore-conf-extensions-postgresql-and-s3.patch new file mode 100644 index 0000000..4462071 --- /dev/null +++ b/patches/conf/0001-chore-conf-extensions-postgresql-and-s3.patch @@ -0,0 +1,39 @@ +From 6c1d545072d80257d57c17c5b15260973bee0330 Mon Sep 17 00:00:00 2001 +From: maxime1907 <19607336+maxime1907@users.noreply.github.com> +Date: Fri, 17 May 2024 10:54:39 +0200 +Subject: [PATCH] chore(conf): extensions: postgresql and s3 + +--- + .../micro-quickstart/_common/common.runtime.properties | 2 +- + .../nano-quickstart/_common/common.runtime.properties | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties b/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties +index b0adb06..c00be36 100644 +--- a/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties ++++ b/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties +@@ -24,7 +24,7 @@ + # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. + # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. + # More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +-druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] ++druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "postgresql-metadata-storage", "druid-s3-extensions"] + + # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory + # and uncomment the line below to point to your directory. +diff --git a/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties b/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties +index b0adb06..c00be36 100644 +--- a/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties ++++ b/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties +@@ -24,7 +24,7 @@ + # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. + # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. + # More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +-druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] ++druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "postgresql-metadata-storage", "druid-s3-extensions"] + + # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory + # and uncomment the line below to point to your directory. +-- +2.34.1 + diff --git a/patches/conf/0002-chore-conf-enable-s3-instead-of-local-storage.patch b/patches/conf/0002-chore-conf-enable-s3-instead-of-local-storage.patch new file mode 100644 index 0000000..5160d50 --- /dev/null +++ b/patches/conf/0002-chore-conf-enable-s3-instead-of-local-storage.patch @@ -0,0 +1,125 @@ +From c244c1121da46cdefc38485056a9fedc0d4239b3 Mon Sep 17 00:00:00 2001 +From: maxime1907 <19607336+maxime1907@users.noreply.github.com> +Date: Fri, 17 May 2024 12:24:07 +0200 +Subject: [PATCH] chore(conf): enable s3 instead of local storage + +--- + .../_common/common.runtime.properties | 27 ++++++++++--------- + .../_common/common.runtime.properties | 27 ++++++++++--------- + 2 files changed, 30 insertions(+), 24 deletions(-) + +diff --git a/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties b/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties +index c00be36..a841ff7 100644 +--- a/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties ++++ b/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties +@@ -77,36 +77,39 @@ druid.metadata.storage.connector.port=1527 + # + + # For local disk (only viable in a cluster if this is a network mount): +-druid.storage.type=local +-druid.storage.storageDirectory=var/druid/segments ++# druid.storage.type=local ++# druid.storage.storageDirectory=var/druid/segments + + # For HDFS: + #druid.storage.type=hdfs + #druid.storage.storageDirectory=/druid/segments + + # For S3: +-#druid.storage.type=s3 +-#druid.storage.bucket=your-bucket +-#druid.storage.baseKey=druid/segments +-#druid.s3.accessKey=... +-#druid.s3.secretKey=... ++druid.storage.type=s3 ++druid.storage.bucket=storage-bucket ++druid.storage.baseKey=druid/segments ++druid.s3.accessKey=access_key ++druid.s3.secretKey=secret_key ++ ++druid.s3.endpoint.url=http://localhost:9000 ++druid.s3.enablePathStyleAccess=true + + # + # Indexing service logs + # + + # For local disk (only viable in a cluster if this is a network mount): +-druid.indexer.logs.type=file +-druid.indexer.logs.directory=var/druid/indexing-logs ++# druid.indexer.logs.type=file ++# druid.indexer.logs.directory=var/druid/indexing-logs + + # For HDFS: + #druid.indexer.logs.type=hdfs + #druid.indexer.logs.directory=/druid/indexing-logs + + # For S3: +-#druid.indexer.logs.type=s3 +-#druid.indexer.logs.s3Bucket=your-bucket +-#druid.indexer.logs.s3Prefix=druid/indexing-logs ++druid.indexer.logs.type=s3 ++druid.indexer.logs.s3Bucket=logs-bucket ++druid.indexer.logs.s3Prefix=druid/indexing-logs + + # + # Service discovery +diff --git a/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties b/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties +index c00be36..a841ff7 100644 +--- a/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties ++++ b/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties +@@ -77,36 +77,39 @@ druid.metadata.storage.connector.port=1527 + # + + # For local disk (only viable in a cluster if this is a network mount): +-druid.storage.type=local +-druid.storage.storageDirectory=var/druid/segments ++# druid.storage.type=local ++# druid.storage.storageDirectory=var/druid/segments + + # For HDFS: + #druid.storage.type=hdfs + #druid.storage.storageDirectory=/druid/segments + + # For S3: +-#druid.storage.type=s3 +-#druid.storage.bucket=your-bucket +-#druid.storage.baseKey=druid/segments +-#druid.s3.accessKey=... +-#druid.s3.secretKey=... ++druid.storage.type=s3 ++druid.storage.bucket=storage-bucket ++druid.storage.baseKey=druid/segments ++druid.s3.accessKey=access_key ++druid.s3.secretKey=secret_key ++ ++druid.s3.endpoint.url=http://localhost:9000 ++druid.s3.enablePathStyleAccess=true + + # + # Indexing service logs + # + + # For local disk (only viable in a cluster if this is a network mount): +-druid.indexer.logs.type=file +-druid.indexer.logs.directory=var/druid/indexing-logs ++# druid.indexer.logs.type=file ++# druid.indexer.logs.directory=var/druid/indexing-logs + + # For HDFS: + #druid.indexer.logs.type=hdfs + #druid.indexer.logs.directory=/druid/indexing-logs + + # For S3: +-#druid.indexer.logs.type=s3 +-#druid.indexer.logs.s3Bucket=your-bucket +-#druid.indexer.logs.s3Prefix=druid/indexing-logs ++druid.indexer.logs.type=s3 ++druid.indexer.logs.s3Bucket=logs-bucket ++druid.indexer.logs.s3Prefix=druid/indexing-logs + + # + # Service discovery +-- +2.34.1 + diff --git a/patches/conf/0003-chore-conf-enable-s3-instead-of-local-storage.patch b/patches/conf/0003-chore-conf-enable-s3-instead-of-local-storage.patch new file mode 100644 index 0000000..432107c --- /dev/null +++ b/patches/conf/0003-chore-conf-enable-s3-instead-of-local-storage.patch @@ -0,0 +1,68 @@ +From eeb18e5b223b6a44bbbffd81560908687d18aca9 Mon Sep 17 00:00:00 2001 +From: maxime1907 <19607336+maxime1907@users.noreply.github.com> +Date: Fri, 17 May 2024 13:56:07 +0200 +Subject: [PATCH] chore(conf): enable s3 instead of local storage + +--- + .../auto/_common/common.runtime.properties | 27 ++++++++++--------- + 1 file changed, 15 insertions(+), 12 deletions(-) + +diff --git a/conf/druid/auto/_common/common.runtime.properties b/conf/druid/auto/_common/common.runtime.properties +index b0adb06..f980465 100644 +--- a/conf/druid/auto/_common/common.runtime.properties ++++ b/conf/druid/auto/_common/common.runtime.properties +@@ -77,36 +77,39 @@ druid.metadata.storage.connector.port=1527 + # + + # For local disk (only viable in a cluster if this is a network mount): +-druid.storage.type=local +-druid.storage.storageDirectory=var/druid/segments ++# druid.storage.type=local ++# druid.storage.storageDirectory=var/druid/segments + + # For HDFS: + #druid.storage.type=hdfs + #druid.storage.storageDirectory=/druid/segments + + # For S3: +-#druid.storage.type=s3 +-#druid.storage.bucket=your-bucket +-#druid.storage.baseKey=druid/segments +-#druid.s3.accessKey=... +-#druid.s3.secretKey=... ++druid.storage.type=s3 ++druid.storage.bucket=storage-bucket ++druid.storage.baseKey=druid/segments ++druid.s3.accessKey=access_key ++druid.s3.secretKey=secret_key ++ ++druid.s3.endpoint.url=http://localhost:9000 ++druid.s3.enablePathStyleAccess=true + + # + # Indexing service logs + # + + # For local disk (only viable in a cluster if this is a network mount): +-druid.indexer.logs.type=file +-druid.indexer.logs.directory=var/druid/indexing-logs ++# druid.indexer.logs.type=file ++# druid.indexer.logs.directory=var/druid/indexing-logs + + # For HDFS: + #druid.indexer.logs.type=hdfs + #druid.indexer.logs.directory=/druid/indexing-logs + + # For S3: +-#druid.indexer.logs.type=s3 +-#druid.indexer.logs.s3Bucket=your-bucket +-#druid.indexer.logs.s3Prefix=druid/indexing-logs ++druid.indexer.logs.type=s3 ++druid.indexer.logs.s3Bucket=logs-bucket ++druid.indexer.logs.s3Prefix=druid/indexing-logs + + # + # Service discovery +-- +2.34.1 + diff --git a/patches/conf/0004-chore-conf-extensions-postgresql-and-s3.patch b/patches/conf/0004-chore-conf-extensions-postgresql-and-s3.patch new file mode 100644 index 0000000..b1bb73d --- /dev/null +++ b/patches/conf/0004-chore-conf-extensions-postgresql-and-s3.patch @@ -0,0 +1,25 @@ +From f6680677305e99d730c45d859bf46c6dae2f7340 Mon Sep 17 00:00:00 2001 +From: maxime1907 <19607336+maxime1907@users.noreply.github.com> +Date: Fri, 17 May 2024 14:00:32 +0200 +Subject: [PATCH] chore(conf): extensions: postgresql and s3 + +--- + conf/druid/auto/_common/common.runtime.properties | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/conf/druid/auto/_common/common.runtime.properties b/conf/druid/auto/_common/common.runtime.properties +index f980465..a841ff7 100644 +--- a/conf/druid/auto/_common/common.runtime.properties ++++ b/conf/druid/auto/_common/common.runtime.properties +@@ -24,7 +24,7 @@ + # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. + # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. + # More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +-druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] ++druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "postgresql-metadata-storage", "druid-s3-extensions"] + + # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory + # and uncomment the line below to point to your directory. +-- +2.34.1 + diff --git a/patches/druid/0001-feat-examples-start-druid-autodetect-cpu-count-in-co.patch b/patches/druid/0001-feat-examples-start-druid-autodetect-cpu-count-in-co.patch new file mode 100644 index 0000000..ceca6b3 --- /dev/null +++ b/patches/druid/0001-feat-examples-start-druid-autodetect-cpu-count-in-co.patch @@ -0,0 +1,110 @@ +From 9d300c1730e5e0b7b3fe3278ca8851ccc315c908 Mon Sep 17 00:00:00 2001 +From: maxime1907 <19607336+maxime1907@users.noreply.github.com> +Date: Fri, 17 May 2024 14:28:35 +0200 +Subject: [PATCH] feat(examples): start-druid: autodetect cpu count in + containers + +--- + bin/start-druid-main.py | 63 +++++++++++++++++++++++++++++--- + 1 file changed, 57 insertions(+), 6 deletions(-) + +diff --git a/bin/start-druid-main.py b/bin/start-druid-main.py +index d797157230..08c6f516c1 100644 +--- a/bin/start-druid-main.py ++++ b/bin/start-druid-main.py +@@ -15,12 +15,13 @@ + # specific language governing permissions and limitations + # under the License. + +-import sys +-import os +-import multiprocessing + import argparse +-import subprocess ++import math ++import multiprocessing ++import os + import platform ++import subprocess ++import sys + + BASE_CONFIG_PATH = "conf/druid/auto" + +@@ -96,6 +97,54 @@ HEAP_TO_TOTAL_MEM_RATIO = { + + LOGGING_ENABLED = False + ++def load(path, encoding="utf-8"): ++ """ Loads a file content """ ++ with open(path, 'r', encoding=encoding, newline="") as handle: ++ tmp = handle.read() ++ return tmp ++ ++def get_cpu_count(): ++ try: ++ try: ++ # This is necessary to deduce docker cpu_count ++ cfs_quota_us = cfs_period_us = 0 ++ # cgroup2 ++ if os.path.exists("/sys/fs/cgroup/cgroup.controllers"): ++ cpu_max = load("/sys/fs/cgroup/cpu.max").split() ++ if cpu_max[0] != "max": ++ if len(cpu_max) == 1: ++ cfs_quota_us, cfs_period_us = int(cpu_max[0]), 100_000 ++ else: ++ cfs_quota_us, cfs_period_us = map(int, cpu_max) ++ else: # cgroup1 ++ cfs_quota_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")) ++ cfs_period_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_period_us")) ++ if cfs_quota_us > 0 and cfs_period_us > 0: ++ return int(math.ceil(cfs_quota_us / cfs_period_us)) ++ except (EnvironmentError, TypeError): ++ pass ++ return multiprocessing.cpu_count() ++ except NotImplementedError: ++ # print("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") ++ return 1 # Safe guess ++ ++def get_memory_limit(): ++ try: ++ # This is necessary to deduce docker memory_limit ++ memory_max = 0 ++ # cgroup2 ++ if os.path.exists("/sys/fs/cgroup/cgroup.controllers"): ++ memory_max = int(load("/sys/fs/cgroup/memory.max")) ++ else: # cgroup1 ++ memory_max = int(load("/sys/fs/cgroup/memory/memory.limit_in_bytes")) ++ if memory_max > 0: ++ return (memory_max/1024/1024) ++ except (EnvironmentError, TypeError): ++ pass ++ physical_memory = get_physical_memory() ++ if physical_memory is None: ++ return 1024 # Safe guess of 1Gb ++ return physical_memory + + def print_if_verbose(message): + if LOGGING_ENABLED: +@@ -391,7 +440,7 @@ def get_physical_memory(): + def convert_total_memory_string(memory): + try: + if memory == '': +- physical_memory = get_physical_memory() ++ physical_memory = get_memory_limit() + + if physical_memory is None: + raise ValueError('Please specify memory argument') +@@ -446,7 +495,9 @@ def build_mm_task_java_opts_array(task_memory): + + + def compute_tasks_memory(allocated_memory): +- cpu_count = multiprocessing.cpu_count() ++ cpu_count = get_cpu_count() ++ ++ print_if_verbose(f'\nDetected CPUs: {cpu_count}') + + if allocated_memory >= cpu_count * 1024: + task_count = cpu_count +-- +2.34.1 + diff --git a/scripts/download_upstream_conf.sh b/scripts/download_upstream_conf.sh new file mode 100755 index 0000000..0baf295 --- /dev/null +++ b/scripts/download_upstream_conf.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +REPO_URL="https://github.com/apache/druid.git" + +BRANCH_OR_TAG="26.0.1" + +CURRENT_DATE=$(date +"%Y-%m-%d-%H-%M-%S") + +CLONE_DIR="/tmp/druid-${BRANCH_OR_TAG}-${CURRENT_DATE}" + +git clone --depth 1 --branch "$BRANCH_OR_TAG" "$REPO_URL" "$CLONE_DIR" + +cp -R "$CLONE_DIR/examples/conf" conf + +git apply patches/conf/*