Spark ML - KMeans - org.apache.spark.sql.AnalysisException：与えられた入力列を `` features` 'で解決できません。

Question

私は、シカゴ犯罪データセットをSpark ML KMeansで分析してクラスタリングしようとしています。下記のスニペット

case class ChicCase(ID: Long, Case_Number: String, Date: String, Block: String, IUCR: String, Primary_Type: String, Description: String, Location_description: String, Arrest: Boolean, Domestic: Boolean, Beat: Int, District: Int, Ward: Int, Community_Area: Int, FBI_Code: String, X_Coordinate: Int, Y_Coordinate: Int, Year: Int, Updated_On: String, Latitude: Double, Longitude: Double, Location: String) 
val city = spark.read.option("header", true).option("inferSchema", true).csv("/chicago_city/Crimes_2001_to_present_2").as[ChicCase] 

val data = city.drop("ID", "Case_Number", "Date", "Block", "IUCR", "Primary_Type", "Description", "Location_description", "Arrest", "Domestic", "FBI_Code", "Year", "Location", "Updated_On") 

val kmeans = new KMeans 
kmeans.setK(10).setSeed(1L) 
val model = kmeans.fit(data) 

しかし、これは

org.apache.spark.sql.AnalysisException: cannot resolve '`features`' given input columns: [Ward, Longitude, X_Coordinate, Beat, Latitude, District, Y_Coordinate, Community_Area]; 
at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42) 
\t at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:77) 
\t at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:74) 
\t at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:301) 
\t at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:301) 
\t at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69) 
at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:300) 
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp$1(QueryPlan.scala:190) 
\t at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:200) 
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2$1.apply(QueryPlan.scala:204) 
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) 
\t at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) 
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) 
\t at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) 
\t at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) 
at scala.collection.AbstractTraversable.map(Traversable.scala:104) 
at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$2(QueryPlan.scala:204) 
\t at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$5.apply(QueryPlan.scala:209) 
\t at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:179) 
\t at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:209) 
\t at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:74) 
\t at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:67) 
\t at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126) 
\t at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67) 
at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:58) 
at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:49) 
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) 
\t at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withPlan(Dataset.scala:2589) 
at org.apache.spark.sql.Dataset.select(Dataset.scala:969) 
at org.apache.spark.ml.clustering.KMeans.fit(KMeans.scala:307) ... 90 elided 

データ型以下の例外をスローしてはintまたはダブルのどちらかであるされています。何が問題なの？

Answer 1

spark mlデータフレームAPIでは、フィーチャーの名前を持つVectorAssemblerを使用して、すべてのフィーチャー列を1つの列として収集する必要があります。 は解決できない「features」与えられた入力列：

import org.apache.spark.ml.feature.VectorAssembler 
import org.apache.spark.ml.clustering.KMeans 

// assembler to collect all interesting columns into a single features column 
val assembler = (new VectorAssembler(). 
        setInputCols(Array("Ward", "Longitude", "X_Coordinate", "Beat", 
             "Latitude", "District", "Y_Coordinate", 
             "Community_Area")). 
        setOutputCol("features")) 

val data = assembler.transform(city)  
val kmeans = new KMeans() 
val model = kmeans.fit(data) 

model.getK 
// res28: Int = 2  example here