本文介紹一些常見的叢集跑hive作業參數優化,可以根據業務需要來使用。
dfs.client.read.shortcircuit=true //直讀
dfs.client.read.shortcircuit.streams.cache.size=4096 //直讀緩存
dfs.datanode.balance.bandwidthPerSec=30048576 //提高balance帶寬,一般擴容後調整
dfs.datanode.max.transfer.threads=16384 //提高線程數
dfs.namenode.checkpoint.period=21600 //延長checkpoint時間
dfs.namenode.handler.count=100 //并發數,大叢集要提高
dfs.namenode.fslock.fair=false //降低寫性能,但提高讀鎖性能
dfs.namenode.lifeline.handler.count=1 //ha叢集優化,大叢集使用
hive.metastore.server.max.threads=100000
hive.compactor.worker.threads=5
hive.metastore.client.socket.timeout=1800s
hive.metastore.failure.retries=5
hive.exec.max.dynamic.partitions=5000
hive.exec.max.dynamic.partitions.pernode=2000
set hive.execution.engine=tez;
SET hive.tez.auto.reducer.parallelism=true;
SET hive.tez.max.partition.factor=20;
STORED AS ORC tblproperties (“orc.compress" = “SNAPPY”)
hive.exec.orc.default.compress=SNAPPY
hive.exec.parallel=true
SET hive.exec.reducers.bytes.per.reducer=128000000;
set hive.vectorized.execution.enabled = true;
set hive.vectorized.execution.reduce.enabled = true;
hive.limit.optimize.enable=true
set hive.cbo.enable=true;
set hive.compute.query.using.stats=true;
set hive.stats.fetch.column.stats=true;
set hive.stats.fetch.partition.stats=true;
查詢前先統計常用表的靜态資訊,常join的列
analyze table tweets compute statistics;
analyze table tweets compute statistics for columns sender, topic;
set hive.enforce.bucketing = <b>true</b>;
set hive.optimize.bucketmapjoin = true;
set hive.optimize.bucketmapjoin.sortedmerge = true;
set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;