广播变量
A broadcast variable. Broadcast variables allow the programmer to keep a read-only variable * cached on each machine rather than shipping a copy of it with tasks. They can be used, for * example, to give every node a copy of a large input dataset in an efficient manner. Spark also * attempts to distribute broadcast variables using efficient broadcast algorithms to reduce * communication cost.
package org.apache.spark.myExamples
import org.apache.spark.sql.SparkSession
/**
* Created by wjf on 16-8-30.
*/
object TestBroadCast {
def main(args: Array[String]): Unit = {
val blockSize = if(args.length >) args() else "4096"
val spark =SparkSession.builder().appName("testBroadCast").config("spark.broadcast.blockSize",blockSize).master("local").getOrCreate()
val sc = spark.sparkContext
val slice = if(args.length >) args().toInt else
val num = if(args.length >) args().toInt else
val arr1=( to num).toArray
for(i <- to ){
println("Iteration " + i)
println("--------------------------")
val start= System.nanoTime()
val broad = sc.broadcast(arr1)
val numSize= sc.parallelize( to ,slice).map(_ => broad.value.length)
numSize.collect().foreach(println)
println("Iteration %d take %.0f milliseconds".format(i,(System.nanoTime()-start)/) )
}
}
}
运行结果
