spark-xgboost8.1 java 例子

标签：register   static   ica   dfs   mode   nts   class   rop   create   

public class Xgb2 implements Serializable{

    /**
     * 
     */
    private static final long serialVersionUID = 1L;

    public static void main(String[] args) throws Exception {       

         Long start=System.currentTimeMillis();
         SparkSession s= SparkSession.builder().appName("XgbTrain").getOrCreate();
         Xgb2 main=new Xgb2();
         main.udf(s);
         main.read(s);
         HttpClientUtil.sendDingMessage("耗时:"+(System.currentTimeMillis()-start)/1000+"s");

    }

    public void udf(SparkSession s) {

         s.udf().register("strToDouble", new UDF1() {
                private static final long serialVersionUID = 1L;

                @Override
                public Double call(String org) throws Exception {
                    return Double.parseDouble(org.trim());
                }
            }, DataTypes.DoubleType);
    }

    public void read(SparkSession s) throws Exception {

        Dataset rs = s.read().csv("hdfs://*/iris.data");

        rs.createOrReplaceTempView("temp");

        rs=s.sql("select strToDouble(_c0) as _c0,  strToDouble(_c1) as _c1, strToDouble(_c2) as _c2, strToDouble(_c3) as _c3, _c4 from temp");

        rs.show(10, false);

        StringIndexerModel stringIndexer = new StringIndexer()
                  .setInputCol("_c4")
                  .setOutputCol("classIndex")
                  .fit(rs);
                // 执行进行转换,并把原有的字符串class删除掉
        rs = stringIndexer.transform(rs).drop("_c4");
                // 将多个字段合并成在一起,组成future
        VectorAssembler vectorAssembler = new VectorAssembler()
                  .setInputCols(new String[] {"_c0", "_c1", "_c2", "_c3"})
                  .setOutputCol("features");
                //将数据集切分成训集和测试集
        rs = vectorAssembler.transform(rs).select("features", "classIndex");
        Dataset[]  splitXgbInput = rs.randomSplit(new double[] {0.9, 0.1});
        Dataset trainXgbInput = splitXgbInput[0];
        Dataset testXgbInput = splitXgbInput[1];

        Map javaMap=new HashMap();
        javaMap.put("objective","multi:softprob");
        javaMap.put("eta",0.1);
        javaMap.put("max_depth",2);
        javaMap.put("num_round","20");
        javaMap.put("num_class",3);

        XGBoostClassifier xgBoostEstimator=new XGBoostClassifier( ConvertUtil.toScalaImmutableMap(javaMap))
                .setFeaturesCol("features").setLabelCol("classIndex");

                // 开始训练
        XGBoostClassificationModel model = xgBoostEstimator.fit(trainXgbInput);
                // 预测
        Dataset result = model.transform(testXgbInput);
                // 展示 
        result.show(10,false);

    }
}

spark-xgboost8.1 java 例子

标签：register   static   ica   dfs   mode   nts   class   rop   create   

原文地址：https://blog.51cto.com/12597095/2492494