dremio 配置文件
作者:神秘网友
发布时间:2021-02-26 16:20:30
dremio 配置文件
因为官方文档对于配置文件的介绍太简单了,比如默认zk 注册的时候会有主机名称的问题，不然我们就需要自己配置
hosts，以下是来自官方代码仓库的一个参考配置，方便使用(注意此配置主要是核心组件的，还有其他组件自己的配置)
配置文件
- 文件位置
源码的common resources dremio-reference.conf - 内容
## Copyright (C) 2017-2019 Dremio Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
include classpath("dremio-reference-ext.conf")
paths: {
# the local path for dremio to store data.
local: "/tmp/dremio",
# the distributed path Dremio data including job results, downloads, uploads, etc
dist: "pdfs://"${paths.local}"/pdfs"
# location for catalog database (if master node)
db: ${paths.local}/db,
spilling: [${paths.local}/spill]
# storage area for the accelerator cache.
accelerator: ${paths.dist}/accelerator
# staging area for json and csv ui downloads
downloads: ${paths.dist}/downloads
# stores uploaded data associated with user home directories
uploads: ${paths.dist}/uploads
# stores data associated with the job results cache.
results: ${paths.dist}/results
# shared scratch space for creation of tables.
scratch: ${paths.dist}/scratch
}
services: {
node-tag: "",
coordinator: {
enabled: true,
# Auto-upgrade Dremio at startup if needed
auto-upgrade: false,
master: {
enabled: true,
# configure an embedded ZooKeeper server on the same node as master
embedded-zookeeper: {
enabled: true,
port: 2181,
path: ${paths.local}/zk
}
},
web: {
enabled: true,
port: 9047,
ssl: {
# If SSL for communication path between browsers (or REST clients) and Dremio should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
},
auth: {
type: "internal"
}
ui: {
# Configuration for Intercom
intercom: {
enabled: true
appid: "@dremio.ui.intercom.appid@"
}
}
tokens: {
cache: {
# number of tokens to store locally on this coordinator node
# (set to 0, if all requests should be made directly to the database of tokens)
size: 100
# time (in minutes) after which the token needs to be obtained from the token store
expiration_minutes: 5
}
}
},
client-endpoint: {
port: 31010
},
scheduler: {
threads: 24
},
command-pool: {
enabled: true
size: 0 # 0 defaults to the machine's number of cores
},
},
executor: {
enabled: true
# enable/disable local cache manager
# storage space for cache manager
# control max percentage of disk cache manager db instance and fs mount points can consume
cache: {
enabled: true,
path: {
db: ${paths.local},
fs: [${services.executor.cache.path.db}]
},
pctquota: {
db: 70,
fs: [${services.executor.cache.pctquota.db}]
},
ensurefreespace: {
fs: [10]
}
}
},
fabric: {
port: 45678,
memory: {
reservation: 100M
}
ssl: {
# If SSL for communication path between Dremio instances should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: false,
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
}
},
flight: {
enabled: true
port: 32010
# Authentication mode to configure the FlightServer Endpoint to use.
# There are two modes:
# - legacy.arrow.flight.auth
# - arrow.flight.auth2
# legacy.arrow.flight.auth is backwards compatible and will be deprecated in the future.
# arrow.flight.auth2 is the new and preferred Arrow Flight authentication method.
auth.mode: "arrow.flight.auth2"
ssl: {
# If SSL for communication path between Flight clients and Dremio should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
}
},
conduit: {
# If set to 0, a port is automatically allocated (typically in ephemeral range). Otherwise, the configured value
# is used.
port: 0
ssl: {
# If SSL for communication path between Dremio instances should be enabled.
enabled: ${services.fabric.ssl.enabled},
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: ${services.fabric.ssl.auto-certificate.enabled},
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${services.fabric.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${services.fabric.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${services.fabric.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${services.fabric.ssl.keyPassword},
# TrustStore type
trustStoreType: ${services.fabric.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${services.fabric.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${services.fabric.ssl.trustStorePassword}
}
}
# Set up kerberos credentials in server (applicable for both coordinator and executor)
kerberos: {
principal: "",
keytab.file.path: ""
}
web-admin: {
enabled: true,
# Port, bound to loopback interface, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)
port: 0
}
jobs: {
# if jobs server should be enabled in a coorinator
enabled: true
}
}
provisioning: {
yarn: {
jvmoptions: "",
# list of jars to be added to the main container classpath
classpath: []
app {
# list of jars to be added to the Dremio application classpath
classpath: []
# list of jars to be added to the Dremio application classpath (at the front)
classpath-prefix: []
}
watchdog: {
# how long to wait for server reply before considering it failed
poll.timeout: 1000ms
# interval between two server polls
poll.interval: 10000ms
# how many consecutive failed attempts before killing server
missed.polls.before.kill: 6
# maximum attempts at killing server
max.kill.attempts: 10
# how long before reattempting killing server
kill.reattempt.interval: 1000ms
}
nodemanager: {
# Enable certificate validation when connecting to Yarn Node-manager REST API
certificate-validation.enabled: true
}
enabled:false
}
}
# the zookeeper quorum for the cluster
zookeeper: "localhost:"${services.coordinator.master.embedded-zookeeper.port}
zk.client.session.timeout: 90000
zk.client.retry.unlimited: true
zk.client.retry.limit: -1
zk.client.retry.initial_timeout_ms: -1
debug: {
enabled: false,
autoPort: false,
prepopulate: false,
singleNode: false,
verboseAccessLog: false,
allowTestApis: false,
forceRemote: false,
useMemoryStorage: false,
addDefaultUser: false,
allowNewerKVStore: true,
# to enable remote debugging of the DremioDaemon running in YARN container
yarnremote.enabled: false
# UI Red Screen Of Death
rsod.enabled: false
# UI File A Bug option
bug.filing.enabled: false
# DX-9126: enable on-idle load shedding
task.on_idle_load_shed: true
# DX-23620: enable eager load shedding
task.eager_load_shed: true
# DX-9911: enable rescheduling task on unblock
task.reschedule_on_unblock: true
# Use election service to elect between multiple master candidates
# has to be set to false if multiple master candidates
master.election.disabled: false,
alloc.max_occupancy_percent: 75,
alloc.est_heap_buf_size_bytes: 800,
# Enable caching for distributed storage, it is turned off by default
dist.caching.enabled: false,
# Max percent of total available cache space to use when possible for distributed storage
dist.max.cache.space.percent: 100,
# Enable asynchronous access for distributed stores when possible
dist.async.enabled: true,
# Enable asynchronous access for uploaded data associated with user home directories
uploads.async.enabled: true,
# Enable asynchronous access for temporary support stores
support.async.enabled: true,
# Enable asynchronous access for the job results cache store
results.async.enabled: true,
# Enable asynchronous access for the create table scratch space
scratch.async.enabled: true,
# Enable asynchronous access for the json and csv download staging space
download.async.enabled: true,
# Enable asynchronous access for the logs store
logs.async.enabled: true,
# NoOp cluster coordinator
noop.cluster.coordinator.enabled: false,
remote.datastore.rpc_timeout_seconds: 60,
dist.s3_file_status_check.enabled: true,
}
# These system properties are listed here to allow substitution of system property values for DAC Web SSL properties
# listed in services.web.ssl section. Currently we consider only the system properties listed in this file for
# substitution.
javax.net.ssl {
keyStoreType: "",
keyStore: "",
keyStorePassword: "",
keyPassword: "",
trustStoreType: "",
trustStore:"",
trustStorePassword: ""
}
registration.publish-host: ""
一个比较有用的配置
registration.publish-host 是一个比较方便的配置，在zk 注册的时候会使用主机名，我们通过此配置可以解决zk 注册主机名的问题
可以方便执行节点的运行（不需要配置hosts了）
dremio 配置文件 相关文章
- nginx开启tls1.2及一些注意问题
因为http传输是明文，通过抓包很容易获取到报文， 所以现在很多站点都开启了https，HTTPS在HTTP的基础上加入了SSL协议，对传输的数据进行加密。 目前主流的ssl协议是tlsv1.2 nginx 配置 https(tlsv1.2)的例子 server_name xxx.xxx.com; listen 443; ssl on;
- Springboot默认配置文件application.properties的常见配置属性
＃SPRING CONFIG（ConfigFileApplicationListener） spring.config.name =＃配置文件名（默认 为 'application' ） spring.config.location =＃配置文件的位置 ＃ 多环境配置文件激活属性spring.profiles.active=dev＃加载application-dev.properties配置文
- nginx配置文件2
server { listen 80; server_name 120.78.64.28; location / {limit_req zone=one ; root /usr/local/nginx/html; index index.html index.htm index.php;#if ($http_user_agent ~* (mobile|nokia|iphone|ipad|android|samsung|htc|blackberry)) { rewrite ^
- spring boot:从yaml配置文件中读取数据的四种方式(spring boot 2.4.3)
一，从配置文件中读取数据有哪些方法 通常有3种用法： 1，直接使用value注解引用得到配置项的值 2, 封装到Component类中再调用: 可以通过value注解或ConfigurationProperties注解两种方式访问 3, 用Environment类从代码中直接访问 生产环境中推荐使用第二种
- c3p0配置Mysql数据源(加载配置文件方式)
jdbc.properties 至于resource jdbc.properties 内容如下 jdbc.driver=com.mysql.jdbc.Driver jdbc.url=jdbc:mysql://localhost:3306/test jdbc.username=root jdbc.password=root @Test public void test1(){ /* * c3p0配置Mysql数据源(加载配置文件方式) *
- uni-app学习笔记（1）配置文件
常用的配置信息 //页面配置 { "pages": [ { "path": "pages/index/index", "style": { "H5":{ } } }, { "path": "pages/login/login", "style": { ... } } ]} //全局样式常用设置 "globalStyle": { "navigationBarTextStyle": "black", |导航栏标题颜色及状
- 自定义字符串匹配度函数 解决字符串相似
因为日志项目名称不同格式（如.xlsx .csv .txt)匹配需要，最近看到一篇比较老的博客 https://www.cnblogs.com/Demcia/p/5453906.html 据此 给出了一个很不错的匹配字符串的思路 // str1= "算法证明" , str2= "个人算法证明复印件"//这里手动分词 ，可以考虑
- dremio sql server 自定义connector开发解决中文乱码问题
在通过反编译查看dremio ce 版本的jdbc 插件，发现对于sql server 的乱码是因为一个配置引起的 所以以下是尝试基于官方的机制，写一个sql server 的扩展，方便集成使用 项目结构 pom.xml xml version="1.0" encoding="UTF-8" project xmlns="http://maven.ap
- yaml语法学习
yaml语法学习 配置文件 SpringBoot使用一个全局的配置文件 ， 配置文件名称是固定的 application.properties 语法结构 ：key=value application.yml 语法结构 ：key：空格 value 配置文件的作用 ： 修改SpringBoot自动配置的默认值，因为SpringBoot在底层都
- ansible 管理 windows
前言 因为工作需求，有一批windows服务器需要管理，因为数量较多，所以研究了一下ansible自动化管理windows服务器，在此记录一下。 环境说明 ansible 服务器：centos 7.2 windows客户端：windows 7 ansible服务器配置 1、需要Python版本为2.7 2、安装pywinrm