Examples with IntDoubleDenseVectorStorage com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage used on opensource projects

Example 1 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class RangeRouterUtils method splitIntDoubleVector.

public static KeyValuePart[] splitIntDoubleVector(MatrixMeta matrixMeta, IntDoubleVector vector) {
    IntDoubleVectorStorage storage = vector.getStorage();
    if (storage.isSparse()) {
        // Get keys and values
        IntDoubleSparseVectorStorage sparseStorage = (IntDoubleSparseVectorStorage) storage;
        int[] keys = sparseStorage.getIndices();
        double[] values = sparseStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, false);
    } else if (storage.isDense()) {
        // Get values
        IntDoubleDenseVectorStorage denseStorage = (IntDoubleDenseVectorStorage) storage;
        double[] values = denseStorage.getValues();
        return split(matrixMeta, vector.getRowId(), values);
    } else {
        // Key and value array pair
        IntDoubleSortedVectorStorage sortStorage = (IntDoubleSortedVectorStorage) storage;
        int[] keys = sortStorage.getIndices();
        double[] values = sortStorage.getValues();
        return split(matrixMeta, vector.getRowId(), keys, values, true);
    }
}

Example 2 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class GBDTController method updateNodeGradStats.

// update node's grad stats on PS
// called during splitting in GradHistHelper, update the grad stats of children nodes after finding the best split
// the root node's stats is updated by leader worker
public void updateNodeGradStats(int nid, GradStats gradStats) throws Exception {
    LOG.debug(String.format("Update gradStats of node[%d]: sumGrad[%f], sumHess[%f]", nid, gradStats.sumGrad, gradStats.sumHess));
    // 1. create the update
    IntDoubleVector vec = new IntDoubleVector(2 * this.activeNode.length, new IntDoubleDenseVectorStorage(2 * this.activeNode.length));
    vec.set(nid, gradStats.sumGrad);
    vec.set(nid + this.activeNode.length, gradStats.sumHess);
    // 2. push the update to PS
    PSModel nodeGradStats = this.model.getPSModel(this.param.nodeGradStatsName);
    nodeGradStats.increment(this.currentTree, vec);
}

Example 3 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class GBDTController method createSketch.

// create data sketch, push candidate split value to PS
public void createSketch() throws Exception {
    PSModel sketch = model.getPSModel(this.param.sketchName);
    PSModel cateFeat = model.getPSModel(this.param.cateFeatureName);
    if (taskContext.getTaskIndex() == 0) {
        LOG.info("------Create sketch------");
        long startTime = System.currentTimeMillis();
        IntDoubleVector sketchVec = new IntDoubleVector(this.param.numFeature * this.param.numSplit, new IntDoubleDenseVectorStorage(new double[this.param.numFeature * this.param.numSplit]));
        IntDoubleVector cateFeatVec = null;
        if (!this.cateFeatList.isEmpty()) {
            cateFeatVec = new IntDoubleVector(this.cateFeatList.size() * this.param.numSplit, new IntDoubleDenseVectorStorage(new double[this.cateFeatList.size() * this.param.numSplit]));
        }
        // 1. calculate candidate split value
        float[][] splits = TYahooSketchSplit.getSplitValue(this.trainDataStore, this.param.numSplit, this.cateFeatList);
        if (splits.length == this.param.numFeature && splits[0].length == this.param.numSplit) {
            for (int fid = 0; fid < splits.length; fid++) {
                if (cateFeatList.contains(fid)) {
                    continue;
                }
                for (int j = 0; j < splits[fid].length; j++) {
                    sketchVec.set(fid * this.param.numSplit + j, splits[fid][j]);
                }
            }
        } else {
            LOG.error("Incompatible sketches size.");
        }
        // categorical features
        if (!this.cateFeatList.isEmpty()) {
            Collections.sort(this.cateFeatList);
            for (int i = 0; i < this.cateFeatList.size(); i++) {
                int fid = this.cateFeatList.get(i);
                int start = i * this.param.numSplit;
                for (int j = 0; j < splits[fid].length; j++) {
                    if (splits[fid][j] == 0 && j > 0)
                        break;
                    cateFeatVec.set(start + j, splits[fid][j]);
                }
            }
        }
        // 2. push local sketch to PS
        sketch.increment(0, sketchVec);
        if (null != cateFeatVec) {
            cateFeat.increment(this.taskContext.getTaskIndex(), cateFeatVec);
        }
        LOG.info(String.format("Create sketch cost: %d ms", System.currentTimeMillis() - startTime));
    }
    Set<String> needFlushMatrixSet = new HashSet<String>(1);
    needFlushMatrixSet.add(this.param.sketchName);
    needFlushMatrixSet.add(this.param.cateFeatureName);
    clockAllMatrix(needFlushMatrixSet, true);
}

Example 4 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class UpdatePSFTest method testDenseDoubleUDF.

public void testDenseDoubleUDF() throws Exception {
    Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
    MatrixClient client1 = worker.getPSAgent().getMatrixClient(DENSE_DOUBLE_MAT, 0);
    int matrixW1Id = client1.getMatrixId();
    int[] index = genIndexs(feaNum, nnz);
    IntDoubleVector deltaVec = new IntDoubleVector(feaNum, new IntDoubleDenseVectorStorage(feaNum));
    for (int i = 0; i < feaNum; i++) {
        deltaVec.set(i, i);
    }
    deltaVec.setRowId(0);
    Vector[] updates = new Vector[1];
    updates[0] = deltaVec;
    client1.asyncUpdate(new IncrementRows(new IncrementRowsParam(matrixW1Id, updates))).get();
    IntDoubleVector row = (IntDoubleVector) client1.getRow(0);
    for (int id : index) {
        Assert.assertEquals(row.get(id), deltaVec.get(id), 0);
    }
    Assert.assertEquals(feaNum, row.size());
}

Example 5 with IntDoubleDenseVectorStorage

use of com.tencent.angel.ml.math2.storage.IntDoubleDenseVectorStorage in project angel by Tencent.

the class MatrixMetaManagerTest method testCreateMatrix.

@Test
public void testCreateMatrix() throws Exception {
    try {
        LOG.info("===========================testCreateMatrix===============================");
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        MasterClient masterClient = worker.getPSAgent().getMasterClient();
        int w3Id = -1;
        int w4Id = -1;
        // add matrix
        MatrixContext mMatrix = new MatrixContext();
        mMatrix.setName("w3");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, RowType.T_DOUBLE_DENSE.name());
        masterClient.createMatrix(mMatrix, 10000);
        mMatrix.setName("w4");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, RowType.T_DOUBLE_DENSE.name());
        masterClient.createMatrix(mMatrix, 10000);
        MatrixMeta w3Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w3");
        MatrixMeta w4Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w4");
        assertEquals(w3Meta.getRowNum(), 1);
        assertEquals(w3Meta.getColNum(), 100000);
        assertEquals(w3Meta.getRowType(), RowType.T_DOUBLE_DENSE);
        assertEquals(w4Meta.getRowNum(), 1);
        assertEquals(w4Meta.getColNum(), 100000);
        assertEquals(w4Meta.getRowType(), RowType.T_DOUBLE_DENSE);
        w3Id = w3Meta.getId();
        w4Id = w4Meta.getId();
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMMatrixMetaManager matrixMetaManager = angelAppMaster.getAppContext().getMatrixMetaManager();
        MatrixMeta matrixw3Proto = matrixMetaManager.getMatrix("w3");
        MatrixMeta matrixw4Proto = matrixMetaManager.getMatrix("w4");
        assertNotNull(matrixw3Proto);
        assertNotNull(matrixw4Proto);
        assertEquals(matrixw3Proto.getRowNum(), 1);
        assertEquals(matrixw3Proto.getColNum(), 100000);
        assertEquals(matrixw3Proto.getPartitionMetas().size(), 2);
        Map<Integer, PartitionMeta> w3Parts = matrixw3Proto.getPartitionMetas();
        assertEquals(w3Parts.get(0).getPss().get(0), psId);
        assertEquals(w3Parts.get(0).getPartId(), 0);
        assertEquals(w3Parts.get(0).getStartRow(), 0);
        assertEquals(w3Parts.get(0).getEndRow(), 1);
        assertEquals(w3Parts.get(0).getStartCol(), 0);
        assertEquals(w3Parts.get(0).getEndCol(), 50000);
        assertEquals(w3Parts.get(1).getPartId(), 1);
        assertEquals(w3Parts.get(1).getStartRow(), 0);
        assertEquals(w3Parts.get(1).getEndRow(), 1);
        assertEquals(w3Parts.get(1).getStartCol(), 50000);
        assertEquals(w3Parts.get(1).getEndCol(), 100000);
        Map<Integer, PartitionMeta> w4Parts = matrixw4Proto.getPartitionMetas();
        assertEquals(w4Parts.get(0).getPss().get(0), psId);
        assertEquals(w4Parts.get(0).getPartId(), 0);
        assertEquals(w4Parts.get(0).getStartRow(), 0);
        assertEquals(w4Parts.get(0).getEndRow(), 1);
        assertEquals(w4Parts.get(0).getStartCol(), 0);
        assertEquals(w4Parts.get(0).getEndCol(), 50000);
        assertEquals(w4Parts.get(1).getPartId(), 1);
        assertEquals(w4Parts.get(1).getStartRow(), 0);
        assertEquals(w4Parts.get(1).getEndRow(), 1);
        assertEquals(w4Parts.get(1).getStartCol(), 50000);
        assertEquals(w4Parts.get(1).getEndCol(), 100000);
        ParameterServer ps = LocalClusterContext.get().getPS(psAttempt0Id).getPS();
        PSMatrixMetaManager matrixPartManager = ps.getMatrixMetaManager();
        PartitionMeta w3Part0 = matrixPartManager.getPartMeta(w3Id, 0);
        PartitionMeta w3Part1 = matrixPartManager.getPartMeta(w3Id, 1);
        assertTrue(w3Part0 != null);
        assertTrue(w3Part1 != null);
        assertEquals(w3Part0.getPartitionKey().getStartRow(), 0);
        assertEquals(w3Part0.getPartitionKey().getEndRow(), 1);
        assertEquals(w3Part0.getPartitionKey().getStartCol(), 0);
        assertEquals(w3Part0.getPartitionKey().getEndCol(), 50000);
        assertEquals(w3Part1.getPartitionKey().getStartRow(), 0);
        assertEquals(w3Part1.getPartitionKey().getEndRow(), 1);
        assertEquals(w3Part1.getPartitionKey().getStartCol(), 50000);
        assertEquals(w3Part1.getPartitionKey().getEndCol(), 100000);
        PartitionMeta w4Part0 = matrixPartManager.getPartMeta(w4Id, 0);
        PartitionMeta w4Part1 = matrixPartManager.getPartMeta(w4Id, 1);
        assertTrue(w4Part0 != null);
        assertTrue(w4Part1 != null);
        assertEquals(w4Part0.getPartitionKey().getStartRow(), 0);
        assertEquals(w4Part0.getPartitionKey().getEndRow(), 1);
        assertEquals(w4Part0.getPartitionKey().getStartCol(), 0);
        assertEquals(w4Part0.getPartitionKey().getEndCol(), 50000);
        assertEquals(w4Part1.getPartitionKey().getStartRow(), 0);
        assertEquals(w4Part1.getPartitionKey().getEndRow(), 1);
        assertEquals(w4Part1.getPartitionKey().getStartCol(), 50000);
        assertEquals(w4Part1.getPartitionKey().getEndCol(), 100000);
        MatrixClient w4ClientForTask0 = worker.getPSAgent().getMatrixClient("w4", 0);
        MatrixClient w4ClientForTask1 = worker.getPSAgent().getMatrixClient("w4", 1);
        TaskContext task0Context = w4ClientForTask0.getTaskContext();
        TaskContext task1Context = w4ClientForTask1.getTaskContext();
        double[] delta = new double[100000];
        for (int i = 0; i < delta.length; i++) {
            delta[i] = 1.0;
        }
        int iterIndex = 0;
        while (iterIndex < 5) {
            IntDoubleVector row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
            double sum1 = sum(row1.getStorage().getValues());
            LOG.info("taskid=" + task0Context.getIndex() + ", matrixId=" + w4ClientForTask0.getMatrixId() + ", rowIndex=0, local row sum=" + sum1);
            IntDoubleVector deltaRow1 = new IntDoubleVector(delta.length, new IntDoubleDenseVectorStorage(delta));
            deltaRow1.setMatrixId(w4ClientForTask0.getMatrixId());
            deltaRow1.setRowId(0);
            w4ClientForTask0.increment(deltaRow1);
            w4ClientForTask0.clock().get();
            task0Context.increaseEpoch();
            IntDoubleVector row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
            double sum2 = sum(row2.getStorage().getValues());
            LOG.info("taskid=" + task1Context.getIndex() + ", matrixId=" + w4ClientForTask1.getMatrixId() + ", rowIndex=1, local row sum=" + sum2);
            IntDoubleVector deltaRow2 = new IntDoubleVector(delta.length, new IntDoubleDenseVectorStorage(delta));
            deltaRow2.setMatrixId(w4ClientForTask1.getMatrixId());
            deltaRow2.setRowId(0);
            w4ClientForTask1.increment(deltaRow2);
            w4ClientForTask1.clock().get();
            task1Context.increaseEpoch();
            iterIndex++;
        }
        AMTaskManager amTaskManager = angelAppMaster.getAppContext().getTaskManager();
        AMTask amTask0 = amTaskManager.getTask(task0Id);
        AMTask amTask1 = amTaskManager.getTask(task1Id);
        assertEquals(amTask0.getIteration(), 5);
        assertEquals(amTask1.getIteration(), 5);
        Int2IntOpenHashMap task0MatrixClocks = amTask0.getMatrixClocks();
        assertEquals(task0MatrixClocks.size(), 1);
        assertEquals(task0MatrixClocks.get(w4Id), 5);
        Int2IntOpenHashMap task1MatrixClocks = amTask1.getMatrixClocks();
        assertEquals(task1MatrixClocks.size(), 1);
        assertEquals(task1MatrixClocks.get(w4Id), 5);
        IntDoubleVector row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
        double sum1 = sum(row1.getStorage().getValues());
        assertEquals(sum1, 1000000.0, 0.000001);
        IntDoubleVector row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
        double sum2 = sum(row2.getStorage().getValues());
        assertEquals(sum2, 1000000.0, 0.000001);
        masterClient.releaseMatrix(w3Meta.getName());
        Thread.sleep(10000);
        matrixw3Proto = matrixMetaManager.getMatrix("w3");
        assertTrue(matrixw3Proto == null);
        MatrixStorageManager matrixStorageManager = LocalClusterContext.get().getPS(psAttempt0Id).getPS().getMatrixStorageManager();
        ServerMatrix sw3 = matrixStorageManager.getMatrix(w3Id);
        assertTrue(sw3 == null);
        w4ClientForTask0.clock().get();
        w4ClientForTask1.clock().get();
        row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
        sum1 = sum(row1.getStorage().getValues());
        assertEquals(sum1, 1000000.0, 0.000001);
        row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
        sum2 = sum(row2.getStorage().getValues());
        assertEquals(sum2, 1000000.0, 0.000001);
    } catch (Exception x) {
        LOG.error("run testCreateMatrix failed ", x);
        throw x;
    }
}