Spark 2.4.0 · CDH 6.3.2 기준 옛 기록(2019년 테스트 됨). 위 버전과 저장소는 오래된 것이라 저장소가 존재하지 않을 수 있다. 현행 절차는 Spark Thrift Server.
bigdata@hdfs-master:~$ wget https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz
bigdata@hdfs-master:~$ vim $SPARK_HOME/sbin/start-thriftserver.sh
export SPARK_HOME=/home/bigdata/apps/spark-2.4.0-bin-hadoop2.7
sbin/start-thriftserver.sh --master yarn --hiveconf hive.server2.thrift.port=10015 --jars "/home/bigdata/igkim/mysql-connector-java-8.0.22.jar"
performance tuning 등 옵션 추가
sbin/start-thriftserver.sh --master yarn --conf spark.sql.thriftServer.incrementalCollect=true --num-executors 5 --executor-cores 4 --executor-memory 16g --driver-cores 4 --driver-memory 8g --hiveconf hive.server2.thrift.port=10015 --jars "/home/bigdata/igkim/mysql-connector-java-8.0.22.jar"
set spark.sql.hive.convertMetastoreParquet=false;
set spark.driver.cores=4;
set spark.driver.memory=16g;
set spark.executor.cores=4;
set spark.executor.memory=16g;
set spark.executor.instances=4;
set spark.executor.memoryOverhead=3g;
set spark.memory.offHeap.enabled=true;
set spark.memory.offHeap.size=16g;
set spark.sql.thriftServer.incrementalCollect=true;
set spark.sql.hive.thriftServer.singleSession=false;
set spark.sql.shuffle.partitions=100;
set spark.sql.inMemoryColumnarStorage.batchSize=50000;
set spark.kryoserializer.buffer.max=1024m;
set spark.sql.hive.convertMetastoreParquet=true;