hadoop - Spark毫升模型保存到HDFS

我试图将我的模型另存为从Spark ml库创建的对象。

但是，这给了我一个错误:

线程“主”中的异常java.lang.NoSuchMethodError:org.apache.spark.ml.PipelineModel.save(Ljava / lang / String;)V
在com.sf.prediction $ .main(prediction.scala:61)
在com.sf.prediction.main(prediction.scala)
在sun.reflect.NativeMethodAccessorImpl.invoke0(本机方法)处
在sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
在sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
在java.lang.reflect.Method.invoke(Method.java:606)
在org.apache.spark.deploy.SparkSubmit $ .org $ apache $ spark $ deploy $ SparkSubmit $$ runMain(SparkSubmit.scala:672)
在org.apache.spark.deploy.SparkSubmit $ .doRunMain $ 1(SparkSubmit.scala:180)
在org.apache.spark.deploy.SparkSubmit $ .submit(SparkSubmit.scala:205)
在org.apache.spark.deploy.SparkSubmit $ .main(SparkSubmit.scala:120)
在org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

以下是我的依赖项:

    <dependency>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest_2.10</artifactId>
        <version>2.1.7</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>2.4.3</version>
        <type>maven-plugin</type>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_2.10</artifactId>
        <version>1.6.0</version>
    </dependency>
    <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-parser-combinators</artifactId>
        <version>2.11.0-M4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_2.10</artifactId>
        <version>1.6.0</version>
    </dependency>

    <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-csv</artifactId>
        <version>1.2</version>
    </dependency>

    <dependency>
        <groupId>com.databricks</groupId>
        <artifactId>spark-csv_2.10</artifactId>
        <version>1.4.0</version>
    </dependency>

    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive_2.10</artifactId>
        <version>1.6.1</version>
    </dependency>

    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-mllib_2.10</artifactId>
        <version>1.6.0</version>
    </dependency>

我还想将模型生成的数据框另存为csv。

model.transform(df).select("features","label","prediction").show()



import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.functions._

import org.apache.spark.SparkConf

import org.apache.spark.sql.hive.HiveContext


import org.apache.spark.ml.feature.OneHotEncoder
import org.apache.spark.ml.feature.VectorAssembler
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.PipelineModel._
import org.apache.spark.ml.feature.{IndexToString, StringIndexer, VectorIndexer}
import org.apache.spark.ml.util.MLWritable

object prediction {
  def main(args: Array[String]): Unit = {

     val conf = new SparkConf()
             .setMaster("local[2]")
             .setAppName("conversion")
    val sc = new SparkContext(conf)

    val hiveContext = new HiveContext(sc)

    val df = hiveContext.sql("select * from prediction_test")
    df.show()
    val credit_indexer = new StringIndexer().setInputCol("transaction_credit_card").setOutputCol("creditCardIndex").fit(df)
    val category_indexer = new StringIndexer().setInputCol("transaction_category").setOutputCol("categoryIndex").fit(df)
    val location_flag_indexer = new StringIndexer().setInputCol("location_flag").setOutputCol("locationIndex").fit(df)
    val label_indexer = new StringIndexer().setInputCol("fraud").setOutputCol("label").fit(df)

    val assembler =  new VectorAssembler().setInputCols(Array("transaction_amount", "creditCardIndex","categoryIndex","locationIndex")).setOutputCol("features")
    val lr = new LogisticRegression().setMaxIter(10).setRegParam(0.01)
    val pipeline = new Pipeline().setStages(Array(credit_indexer, category_indexer, location_flag_indexer, label_indexer, assembler, lr))

    val model = pipeline.fit(df)

    pipeline.save("/user/f42h/prediction/pipeline")
    model.save("/user/f42h/prediction/model")
 //   val sameModel = PipelineModel.load("/user/bob/prediction/model")
    model.transform(df).select("features","label","prediction")

  }
}

最佳答案

您使用的是Spark 1.6.0和afaik，ml模型的保存/加载仅从2.0开始可用。您可以使用带有2.0.0-preview版本的 Artifact 的预览:http://search.maven.org/#search%7Cga%7C1%7Cg%3Aorg.apache.spark%20v%3A2.0.0-preview