Examples with MultivariateGaussian com.alibaba.alink.operator.common.statistics.basicstatistic.MultivariateGaussian used on opensource projects

Example 1 with MultivariateGaussian

use of com.alibaba.alink.operator.common.statistics.basicstatistic.MultivariateGaussian in project Alink by alibaba.

the class GmmModelMapper method loadModel.

@Override
public void loadModel(List<Row> modelRows) {
    this.modelData = new GmmModelDataConverter().load(modelRows);
    this.multivariateGaussians = new MultivariateGaussian[this.modelData.k];
    for (int i = 0; i < this.modelData.k; i++) {
        this.multivariateGaussians[i] = new MultivariateGaussian(modelData.data.get(i).mean, GmmModelData.expandCovarianceMatrix(modelData.data.get(i).cov, modelData.dim));
    }
    threadLocalProb = ThreadLocal.withInitial(() -> new double[this.modelData.k]);
}

Example 2 with MultivariateGaussian

use of com.alibaba.alink.operator.common.statistics.basicstatistic.MultivariateGaussian in project Alink by alibaba.

the class GmmTrainBatchOp method linkFrom.

/**
 * Train the Gaussian Mixture model with Expectation-Maximization algorithm.
 */
@Override
public GmmTrainBatchOp linkFrom(BatchOperator<?>... inputs) {
    BatchOperator<?> in = checkAndGetFirst(inputs);
    final String vectorColName = getVectorCol();
    final int numClusters = getK();
    final int maxIter = getMaxIter();
    final double tol = getEpsilon();
    // Extract the vectors from the input operator.
    Tuple2<DataSet<Vector>, DataSet<BaseVectorSummary>> vectorAndSummary = StatisticsHelper.summaryHelper(in, null, vectorColName);
    DataSet<Integer> featureSize = vectorAndSummary.f1.map(new MapFunction<BaseVectorSummary, Integer>() {

        private static final long serialVersionUID = 8456872852742625845L;

        @Override
        public Integer map(BaseVectorSummary summary) throws Exception {
            return summary.vectorSize();
        }
    });
    DataSet<Vector> data = vectorAndSummary.f0.map(new RichMapFunction<Vector, Vector>() {

        private static final long serialVersionUID = -845795862675993897L;

        transient int featureSize;

        @Override
        public void open(Configuration parameters) throws Exception {
            List<Integer> bc = getRuntimeContext().getBroadcastVariable("featureSize");
            this.featureSize = bc.get(0);
        }

        @Override
        public Vector map(Vector vec) throws Exception {
            if (vec instanceof SparseVector) {
                ((SparseVector) vec).setSize(featureSize);
            }
            return vec;
        }
    }).withBroadcastSet(featureSize, "featureSize");
    // Initialize the model.
    DataSet<Tuple3<Integer, GmmClusterSummary, IterationStatus>> initialModel = initRandom(data, numClusters, getRandomSeed());
    // Iteratively update the model with EM algorithm.
    IterativeDataSet<Tuple3<Integer, GmmClusterSummary, IterationStatus>> loop = initialModel.iterate(maxIter);
    DataSet<Tuple4<Integer, GmmClusterSummary, IterationStatus, MultivariateGaussian>> md = loop.mapPartition(new RichMapPartitionFunction<Tuple3<Integer, GmmClusterSummary, IterationStatus>, Tuple4<Integer, GmmClusterSummary, IterationStatus, MultivariateGaussian>>() {

        private static final long serialVersionUID = -1937088240477952410L;

        @Override
        public void mapPartition(Iterable<Tuple3<Integer, GmmClusterSummary, IterationStatus>> values, Collector<Tuple4<Integer, GmmClusterSummary, IterationStatus, MultivariateGaussian>> collector) throws Exception {
            for (Tuple3<Integer, GmmClusterSummary, IterationStatus> value : values) {
                DenseVector means = value.f1.mean;
                DenseMatrix cov = GmmModelData.expandCovarianceMatrix(value.f1.cov, means.size());
                MultivariateGaussian md = new MultivariateGaussian(means, cov);
                collector.collect(Tuple4.of(value.f0, value.f1, value.f2, md));
            }
        }
    }).withForwardedFields("f0;f1;f2");
    DataSet<Tuple3<Integer, GmmClusterSummary, IterationStatus>> updatedModel = data.<LocalAggregator>mapPartition(new RichMapPartitionFunction<Vector, LocalAggregator>() {

        private static final long serialVersionUID = 8356493076036649604L;

        transient DenseVector oldWeights;

        transient DenseVector[] oldMeans;

        transient DenseVector[] oldCovs;

        transient MultivariateGaussian[] mnd;

        @Override
        public void open(Configuration parameters) throws Exception {
            oldWeights = new DenseVector(numClusters);
            oldMeans = new DenseVector[numClusters];
            oldCovs = new DenseVector[numClusters];
            mnd = new MultivariateGaussian[numClusters];
        }

        @Override
        public void mapPartition(Iterable<Vector> values, Collector<LocalAggregator> out) throws Exception {
            List<Integer> bcNumFeatures = getRuntimeContext().getBroadcastVariable("featureSize");
            List<Tuple4<Integer, GmmClusterSummary, IterationStatus, MultivariateGaussian>> bcOldModel = getRuntimeContext().getBroadcastVariable("oldModel");
            double prevLogLikelihood = 0.;
            for (Tuple4<Integer, GmmClusterSummary, IterationStatus, MultivariateGaussian> t : bcOldModel) {
                int clusterId = t.f0;
                GmmClusterSummary clusterInfo = t.f1;
                prevLogLikelihood = t.f2.currLogLikelihood;
                oldWeights.set(clusterId, clusterInfo.weight);
                oldMeans[clusterId] = clusterInfo.mean;
                oldCovs[clusterId] = clusterInfo.cov;
                mnd[clusterId] = new MultivariateGaussian(t.f3);
            // mnd[clusterId] = t.f3;
            }
            LocalAggregator aggregator = new LocalAggregator(numClusters, bcNumFeatures.get(0), prevLogLikelihood, oldWeights, oldMeans, oldCovs, mnd);
            values.forEach(aggregator::add);
            out.collect(aggregator);
        }
    }).withBroadcastSet(featureSize, "featureSize").withBroadcastSet(md, "oldModel").name("E-M_step").reduce(new ReduceFunction<LocalAggregator>() {

        private static final long serialVersionUID = -6976429920344470952L;

        @Override
        public LocalAggregator reduce(LocalAggregator value1, LocalAggregator value2) throws Exception {
            return value1.merge(value2);
        }
    }).flatMap(new RichFlatMapFunction<LocalAggregator, Tuple3<Integer, GmmClusterSummary, IterationStatus>>() {

        private static final long serialVersionUID = 6599047947335456972L;

        @Override
        public void flatMap(LocalAggregator aggregator, Collector<Tuple3<Integer, GmmClusterSummary, IterationStatus>> out) throws Exception {
            for (int i = 0; i < numClusters; i++) {
                double w = aggregator.updatedWeightsSum.get(i);
                aggregator.updatedMeansSum[i].scaleEqual(1.0 / w);
                aggregator.updatedCovsSum[i].scaleEqual(1.0 / w);
                GmmClusterSummary model = new GmmClusterSummary(i, w / aggregator.totalCount, aggregator.updatedMeansSum[i], aggregator.updatedCovsSum[i]);
                // note that we use Cov(X,Y) = E[XY] - E[X]E[Y] to compute Cov(X,Y)
                int featureSize = model.mean.size();
                for (int m = 0; m < featureSize; m++) {
                    // loop over columns
                    for (int n = m; n < featureSize; n++) {
                        int pos = GmmModelData.getElementPositionInCompactMatrix(m, n, featureSize);
                        model.cov.add(pos, -1.0 * model.mean.get(m) * model.mean.get(n));
                    }
                }
                IterationStatus stat = new IterationStatus();
                stat.prevLogLikelihood = aggregator.prevLogLikelihood;
                stat.currLogLikelihood = aggregator.newLogLikelihood;
                out.collect(Tuple3.of(i, model, stat));
            }
        }
    }).partitionCustom(new Partitioner<Integer>() {

        private static final long serialVersionUID = 1006932050560340472L;

        @Override
        public int partition(Integer key, int numPartitions) {
            return key % numPartitions;
        }
    }, 0);
    // Check whether stop criterion is met.
    DataSet<Boolean> criterion = updatedModel.first(1).flatMap(new RichFlatMapFunction<Tuple3<Integer, GmmClusterSummary, IterationStatus>, Boolean>() {

        private static final long serialVersionUID = 6890280483282243057L;

        @Override
        public void flatMap(Tuple3<Integer, GmmClusterSummary, IterationStatus> value, Collector<Boolean> out) throws Exception {
            IterationStatus stat = value.f2;
            int stepNo = getIterationRuntimeContext().getSuperstepNumber();
            double diffLogLikelihood = Math.abs(stat.currLogLikelihood - stat.prevLogLikelihood);
            LOG.info("step {}, prevLogLikelihood {}, currLogLikelihood {}, diffLogLikelihood {}", stepNo, stat.prevLogLikelihood, stat.currLogLikelihood, diffLogLikelihood);
            if (stepNo <= 1 || diffLogLikelihood > tol) {
                out.collect(false);
            }
        }
    });
    DataSet<Tuple3<Integer, GmmClusterSummary, IterationStatus>> finalModel = loop.closeWith(updatedModel, criterion);
    // Output the model.
    DataSet<Row> modelRows = finalModel.mapPartition(new RichMapPartitionFunction<Tuple3<Integer, GmmClusterSummary, IterationStatus>, Row>() {

        private static final long serialVersionUID = -8411238421923712023L;

        transient int featureSize;

        @Override
        public void open(Configuration parameters) throws Exception {
            this.featureSize = (int) (getRuntimeContext().getBroadcastVariable("featureSize").get(0));
        }

        @Override
        public void mapPartition(Iterable<Tuple3<Integer, GmmClusterSummary, IterationStatus>> values, Collector<Row> out) throws Exception {
            int numTasks = getRuntimeContext().getNumberOfParallelSubtasks();
            if (numTasks > 1) {
                throw new RuntimeException("parallelism is not 1 when saving model.");
            }
            GmmModelData model = new GmmModelData();
            model.k = numClusters;
            model.dim = featureSize;
            model.vectorCol = vectorColName;
            model.data = new ArrayList<>(numClusters);
            for (Tuple3<Integer, GmmClusterSummary, IterationStatus> t : values) {
                t.f1.clusterId = t.f0;
                model.data.add(t.f1);
            }
            new GmmModelDataConverter().save(model, out);
        }
    }).setParallelism(1).withBroadcastSet(featureSize, "featureSize");
    this.setOutput(modelRows, new GmmModelDataConverter().getModelSchema());
    return this;
}