Spark計算相關性系數（皮爾森、斯皮爾曼、卡方檢驗）

皮爾森、斯皮爾曼（pearson spearman）：

import spark.implicits._
   import org.apache.spark.mllib.stat.Statistics
   import spark.sql  
   val df = sql(s"select * from xxxx ")
    
   val columns = List("xx","xx","xx")
   for(col <- columns){
      
     val df_real = df.select("label", col) 
     val rdd_real = df_real.rdd.map(x=>(x(0).toString.toDouble ,x(1).toString.toDouble ))
     val label = rdd_real.map(x=>x._1.toDouble )
     val feature = rdd_real.map(x=>x._2.toDouble )
     
     val cor_pearson:Double = Statistics.corr(label, feature, "pearson")
     println( s"${col}------" + cor_pearson )
 
     val cor_spearman:Double = Statistics.corr(label, feature, "spearman")
     println(s"${col}------" + cor_spearman )
   }
           

卡方檢驗計算卡方值：

import org.apache.spark.mllib.linalg.{Matrix, Matrices, Vectors }
   import org.apache.spark.mllib.regression.LabeledPoint 
   import org.apache.spark.mllib.stat.Statistics
   import spark.implicits._ 
   import spark.sql
   val df_real = sql(s"select * from  xxxx ")
    
    val columns = List("xx", "xx","xx","xx" )
  
  val featInd = columns.map(df_real.columns.indexOf(_))
  val targetInd = df_real.columns.indexOf("label") 
  val lp_data = df_real.rdd.map(r => LabeledPoint(
   r.getString(targetInd).toDouble,   
   Vectors.dense(featInd.map(r.getString(_).toDouble).toArray) 
))
   val vd=Statistics.chiSqTest(lp_data) 
     
  vd.foreach(x=>println(x.statistic))
  columns.foreach(println(_))