Examples with ReduceScatter com.alibaba.alink.operator.common.tree.parallelcart.communication.ReduceScatter used on opensource projects

Example 1 with ReduceScatter

use of com.alibaba.alink.operator.common.tree.parallelcart.communication.ReduceScatter in project Alink by alibaba.

the class BaseGbdtTrainBatchOp method linkFrom.

@Override
public T linkFrom(BatchOperator<?>... inputs) {
    BatchOperator<?> in = checkAndGetFirst(inputs);
    LOG.info("gbdt train start");
    if (!Preprocessing.isSparse(getParams())) {
        getParams().set(HasCategoricalCols.CATEGORICAL_COLS, TableUtil.getCategoricalCols(in.getSchema(), getParams().get(GbdtTrainParams.FEATURE_COLS), getParams().contains(GbdtTrainParams.CATEGORICAL_COLS) ? getParams().get(GbdtTrainParams.CATEGORICAL_COLS) : null));
    }
    LossType loss = getParams().get(LossUtils.LOSS_TYPE);
    getParams().set(ALGO_TYPE, LossUtils.lossTypeToInt(loss));
    rewriteLabelType(in.getSchema(), getParams());
    if (!Preprocessing.isSparse(getParams())) {
        getParams().set(ModelParamName.FEATURE_TYPES, FlinkTypeConverter.getTypeString(TableUtil.findColTypes(in.getSchema(), getParams().get(GbdtTrainParams.FEATURE_COLS))));
    }
    if (LossUtils.isRanking(getParams().get(LossUtils.LOSS_TYPE))) {
        if (!getParams().contains(LambdaMartNdcgParams.GROUP_COL)) {
            throw new IllegalArgumentException("Group column should be set in ranking loss function.");
        }
    }
    String[] trainColNames = trainColsWithGroup();
    // check label if has null value or not.
    final String labelColName = this.getParams().get(HasLabelCol.LABEL_COL);
    final int labelColIdx = TableUtil.findColIndex(in.getSchema(), labelColName);
    in = new TableSourceBatchOp(DataSetConversionUtil.toTable(in.getMLEnvironmentId(), in.getDataSet().map(new MapFunction<Row, Row>() {

        @Override
        public Row map(Row row) throws Exception {
            if (null == row.getField(labelColIdx)) {
                throw new RuntimeException("label col has null values.");
            }
            return row;
        }
    }), in.getSchema())).setMLEnvironmentId(in.getMLEnvironmentId());
    in = Preprocessing.select(in, trainColNames);
    DataSet<Object[]> labels = Preprocessing.generateLabels(in, getParams(), LossUtils.isRegression(loss) || LossUtils.isRanking(loss));
    if (LossUtils.isClassification(loss)) {
        labels = labels.map(new CheckNumLabels4BinaryClassifier());
    }
    DataSet<Row> trainDataSet;
    BatchOperator<?> stringIndexerModel;
    BatchOperator<?> quantileModel;
    if (getParams().get(USE_ONEHOT)) {
        // create empty string indexer model.
        stringIndexerModel = Preprocessing.generateStringIndexerModel(in, new Params());
        // create empty quantile model.
        quantileModel = Preprocessing.generateQuantileDiscretizerModel(in, new Params().set(HasFeatureCols.FEATURE_COLS, new String[] {}).set(HasCategoricalCols.CATEGORICAL_COLS, new String[] {}));
        trainDataSet = Preprocessing.castLabel(in, getParams(), labels, LossUtils.isRegression(loss) || LossUtils.isRanking(loss)).getDataSet();
    } else if (getParams().get(USE_EPSILON_APPRO_QUANTILE)) {
        // create string indexer model
        stringIndexerModel = Preprocessing.generateStringIndexerModel(in, getParams());
        // create empty quantile model
        quantileModel = Preprocessing.generateQuantileDiscretizerModel(in, new Params().set(HasFeatureCols.FEATURE_COLS, new String[] {}).set(HasCategoricalCols.CATEGORICAL_COLS, new String[] {}));
        trainDataSet = Preprocessing.castLabel(Preprocessing.isSparse(getParams()) ? in : Preprocessing.castContinuousCols(Preprocessing.castCategoricalCols(in, stringIndexerModel, getParams()), getParams()), getParams(), labels, LossUtils.isRegression(loss) || LossUtils.isRanking(loss)).getDataSet();
    } else {
        stringIndexerModel = Preprocessing.generateStringIndexerModel(in, getParams());
        quantileModel = Preprocessing.generateQuantileDiscretizerModel(in, getParams());
        trainDataSet = Preprocessing.castLabel(Preprocessing.castToQuantile(Preprocessing.isSparse(getParams()) ? in : Preprocessing.castContinuousCols(Preprocessing.castCategoricalCols(in, stringIndexerModel, getParams()), getParams()), quantileModel, getParams()), getParams(), labels, LossUtils.isRegression(loss) || LossUtils.isRanking(loss)).getDataSet();
    }
    if (LossUtils.isRanking(getParams().get(LossUtils.LOSS_TYPE))) {
        trainDataSet = trainDataSet.partitionCustom(new Partitioner<Number>() {

            private static final long serialVersionUID = -7790649477852624964L;

            @Override
            public int partition(Number key, int numPartitions) {
                return (int) (key.longValue() % numPartitions);
            }
        }, 0);
    }
    DataSet<Tuple2<Double, Long>> sum = trainDataSet.mapPartition(new MapPartitionFunction<Row, Tuple2<Double, Long>>() {

        private static final long serialVersionUID = -8333738060239409640L;

        @Override
        public void mapPartition(Iterable<Row> iterable, Collector<Tuple2<Double, Long>> collector) throws Exception {
            double sum = 0.;
            long cnt = 0;
            for (Row row : iterable) {
                sum += ((Number) row.getField(row.getArity() - 1)).doubleValue();
                cnt++;
            }
            collector.collect(Tuple2.of(sum, cnt));
        }
    }).reduce(new ReduceFunction<Tuple2<Double, Long>>() {

        private static final long serialVersionUID = -6464200385237876961L;

        @Override
        public Tuple2<Double, Long> reduce(Tuple2<Double, Long> t0, Tuple2<Double, Long> t1) throws Exception {
            return Tuple2.of(t0.f0 + t1.f0, t0.f1 + t1.f1);
        }
    });
    DataSet<FeatureMeta> featureMetas;
    if (getParams().get(USE_ONEHOT)) {
        featureMetas = DataUtil.createOneHotFeatureMeta(trainDataSet, getParams(), trainColNames);
    } else if (getParams().get(USE_EPSILON_APPRO_QUANTILE)) {
        featureMetas = DataUtil.createEpsilonApproQuantileFeatureMeta(trainDataSet, stringIndexerModel.getDataSet(), getParams(), trainColNames, getMLEnvironmentId());
    } else {
        featureMetas = DataUtil.createFeatureMetas(quantileModel.getDataSet(), stringIndexerModel.getDataSet(), getParams());
    }
    {
        getParams().set(BoosterType.BOOSTER_TYPE, BoosterType.HESSION_BASE);
        getParams().set(CriteriaType.CRITERIA_TYPE, CriteriaType.valueOf(getParams().get(GbdtTrainParams.CRITERIA).toString()));
        if (getParams().get(GbdtTrainParams.NEWTON_STEP)) {
            getParams().set(LeafScoreUpdaterType.LEAF_SCORE_UPDATER_TYPE, LeafScoreUpdaterType.NEWTON_SINGLE_STEP_UPDATER);
        } else {
            getParams().set(LeafScoreUpdaterType.LEAF_SCORE_UPDATER_TYPE, LeafScoreUpdaterType.WEIGHT_AVG_UPDATER);
        }
    }
    IterativeComQueue comQueue = new IterativeComQueue().initWithPartitionedData("trainData", trainDataSet).initWithBroadcastData("gbdt.y.sum", sum).initWithBroadcastData("quantileModel", quantileModel.getDataSet()).initWithBroadcastData("stringIndexerModel", stringIndexerModel.getDataSet()).initWithBroadcastData("labels", labels).initWithBroadcastData("featureMetas", featureMetas).add(new InitBoostingObjs(getParams())).add(new Boosting()).add(new Bagging()).add(new InitTreeObjs());
    if (getParams().get(USE_EPSILON_APPRO_QUANTILE)) {
        comQueue.add(new BuildLocalSketch()).add(new AllReduceT<>(BuildLocalSketch.SKETCH, BuildLocalSketch.FEATURE_SKETCH_LENGTH, new BuildLocalSketch.SketchReducer(getParams()), EpsilonApproQuantile.WQSummary.class)).add(new FinalizeBuildSketch());
    }
    comQueue.add(new ConstructLocalHistogram()).add(new ReduceScatter("histogram", "histogram", "recvcnts", AllReduce.SUM)).add(new CalcFeatureGain()).add(new AllReduceT<>("best", "bestLength", new NodeReducer(), Node.class)).add(new SplitInstances()).add(new UpdateLeafScore()).add(new UpdatePredictionScore()).setCompareCriterionOfNode0(new TerminateCriterion()).closeWith(new SaveModel(getParams()));
    DataSet<Row> model = comQueue.exec();
    setOutput(model, new TreeModelDataConverter(FlinkTypeConverter.getFlinkType(getParams().get(ModelParamName.LABEL_TYPE_NAME))).getModelSchema());
    this.setSideOutputTables(new Table[] { DataSetConversionUtil.toTable(getMLEnvironmentId(), model.reduceGroup(new TreeModelDataConverter.FeatureImportanceReducer()), new String[] { getParams().get(TreeModelDataConverter.IMPORTANCE_FIRST_COL), getParams().get(TreeModelDataConverter.IMPORTANCE_SECOND_COL) }, new TypeInformation[] { Types.STRING, Types.DOUBLE }) });
    return (T) this;
}