Examples with MatrixClient com.tencent.angel.psagent.matrix.MatrixClient used on opensource projects

Example 11 with MatrixClient

use of com.tencent.angel.psagent.matrix.MatrixClient in project angel by Tencent.

the class PSFTestTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient client = taskContext.getMatrix("psf_test");
        Pull func = new Pull(client.getMatrixId(), 0);
        Pull func1 = new Pull(client.getMatrixId(), 1);
        while (taskContext.getEpoch() < 100) {
            taskContext.globalSync(client.getMatrixId());
            long startTs = System.currentTimeMillis();
            TVector row = ((GetRowResult) client.get(func)).getRow();
            TVector row1 = ((GetRowResult) client.get(func1)).getRow();
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " pull use time=" + (System.currentTimeMillis() - startTs) + ", sum of row 0=" + sum((DenseDoubleVector) row) + " sum of row 1=" + sum((DenseDoubleVector) row1));
            double[] delta = new double[10000000];
            for (int i = 0; i < 10000000; i++) {
                delta[i] = 1.0;
            }
            DenseDoubleVector deltaV = new DenseDoubleVector(10000000, delta);
            deltaV.setMatrixId(client.getMatrixId());
            deltaV.setRowId(0);
            double[] delta1 = new double[10000000];
            for (int i = 0; i < 10000000; i++) {
                delta1[i] = 2.0;
            }
            DenseDoubleVector deltaV1 = new DenseDoubleVector(10000000, delta1);
            deltaV1.setMatrixId(client.getMatrixId());
            deltaV1.setRowId(1);
            client.increment(deltaV);
            client.increment(deltaV1);
            client.clock().get();
            taskContext.incEpoch();
        }
    } catch (Throwable x) {
        throw new AngelException("run task failed ", x);
    }
}

Example 12 with MatrixClient

use of com.tencent.angel.psagent.matrix.MatrixClient in project angel by Tencent.

the class SparseDoubleTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient client = taskContext.getMatrix("sparse_double_test");
        while (taskContext.getEpoch() < 100) {
            long startTs = System.currentTimeMillis();
            TVector row = client.getRow(0);
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " pull use time=" + (System.currentTimeMillis() - startTs) + ", sum=" + sum((SparseDoubleVector) row));
            startTs = System.currentTimeMillis();
            SparseDoubleVector deltaV = new SparseDoubleVector(2100000000, 150000000);
            for (int i = 0; i < 2100000000; i += 20) {
                deltaV.set(i, 1.0);
            }
            deltaV.setMatrixId(client.getMatrixId());
            deltaV.setRowId(0);
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " train use time=" + (System.currentTimeMillis() - startTs));
            startTs = System.currentTimeMillis();
            client.increment(deltaV);
            client.clock().get();
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " flush use time=" + (System.currentTimeMillis() - startTs));
            taskContext.incEpoch();
        }
    } catch (Throwable x) {
        throw new AngelException("run task failed ", x);
    }
}

Example 13 with MatrixClient

use of com.tencent.angel.psagent.matrix.MatrixClient in project angel by Tencent.

the class PSFailedReportTest method testPSFailedReport.

@Test
public void testPSFailedReport() throws Exception {
    ParameterServerId ps1Id = new ParameterServerId(0);
    final ParameterServerId ps2Id = new ParameterServerId(1);
    PSAttemptId ps1Attempt0Id = new PSAttemptId(ps1Id, 0);
    PSAttemptId ps2Attempt0Id = new PSAttemptId(ps2Id, 0);
    PSAttemptId ps2Attempt1Id = new PSAttemptId(ps2Id, 1);
    ParameterServer ps1Attempt0 = LocalClusterContext.get().getPS(ps1Attempt0Id).getPS();
    ParameterServer ps2Attempt0 = LocalClusterContext.get().getPS(ps2Attempt0Id).getPS();
    WorkerId worker0Id = new WorkerId(new WorkerGroupId(0), 0);
    WorkerAttemptId worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    Worker worker0 = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
    TaskContext task0Context = worker0.getTaskManager().getRunningTask().get(task0Id).getTaskContext();
    MatrixClient matrixClient = task0Context.getMatrix("w1");
    int iterNum = 20;
    for (int i = 0; i < iterNum; i++) {
        DenseIntVector update = new DenseIntVector(dim);
        for (int j = 0; j < dim; j++) {
            update.set(j, 1);
        }
        update.setMatrixId(matrixClient.getMatrixId());
        update.setRowId(0);
        matrixClient.increment(update);
        matrixClient.clock().get();
        Thread.sleep(1000);
        MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
        ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps1w1.getPartition(0));
        assertNotNull(ps1w1.getPartition(1));
        IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
        int part0Size = ps1w1.getRow(0, 0).size();
        IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
        int part1Size = ps1w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
        MatrixStorageManager ps2Storage = ps2Attempt0.getMatrixStorageManager();
        ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps2w1.getPartition(0));
        assertNotNull(ps2w1.getPartition(1));
        row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
        part0Size = ps2w1.getRow(0, 0).size();
        row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
        part1Size = ps2w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
    }
    LOG.info("===================================================================ps2 failed");
    HashMap<PSLocation, Integer> failedCounters = new HashMap<>();
    PSLocation psLoc = new PSLocation(ps2Id, ps2Attempt0.getLocationManager().getPsLocation(ps2Id));
    failedCounters.put(psLoc, 10000);
    worker0.getPSAgent().getMasterClient().psFailedReport(failedCounters);
    Thread.sleep(20000);
    for (int i = iterNum; i < 2 * iterNum; i++) {
        DenseIntVector update = new DenseIntVector(dim);
        for (int j = 0; j < dim; j++) {
            update.set(j, 1);
        }
        update.setMatrixId(matrixClient.getMatrixId());
        update.setRowId(0);
        matrixClient.increment(update);
        matrixClient.clock().get();
        Thread.sleep(1000);
        MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
        ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps1w1.getPartition(0));
        assertNotNull(ps1w1.getPartition(1));
        IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
        int part0Size = ps1w1.getRow(0, 0).size();
        IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
        int part1Size = ps1w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
    }
    ParameterServer ps2Attempt = LocalClusterContext.get().getPS(ps2Attempt1Id).getPS();
    for (int i = iterNum * 2; i < 3 * iterNum; i++) {
        DenseIntVector update = new DenseIntVector(dim);
        for (int j = 0; j < dim; j++) {
            update.set(j, 1);
        }
        update.setMatrixId(matrixClient.getMatrixId());
        update.setRowId(0);
        matrixClient.increment(update);
        matrixClient.clock().get();
        Thread.sleep(1000);
        MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
        ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps1w1.getPartition(0));
        assertNotNull(ps1w1.getPartition(1));
        IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
        int part0Size = ps1w1.getRow(0, 0).size();
        IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
        int part1Size = ps1w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
        MatrixStorageManager ps2Storage = ps2Attempt.getMatrixStorageManager();
        ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps2w1.getPartition(0));
        assertNotNull(ps2w1.getPartition(1));
        row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
        part0Size = ps2w1.getRow(0, 0).size();
        row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
        part1Size = ps2w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
    }
}

Example 14 with MatrixClient

use of com.tencent.angel.psagent.matrix.MatrixClient in project angel by Tencent.

the class IndexGetRowHashTest method testSparseFloatUDF.

public void testSparseFloatUDF() throws Exception {
    Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
    MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_FLOAT_MAT, 0);
    int matrixW1Id = client1.getMatrixId();
    int[] index = genIndexs(feaNum, nnz);
    IntFloatVector deltaVec = new IntFloatVector(feaNum, new IntFloatSparseVectorStorage(feaNum, nnz));
    for (int i = 0; i < index.length; i++) {
        deltaVec.set(index[i], index[i]);
    }
    // for (int i = 0; i < feaNum; i++) {
    // deltaVec.set(i, i);
    // }
    deltaVec.setRowId(0);
    client1.increment(deltaVec);
    client1.clock().get();
    // IndexGet func = new IndexGet(new IndexGetParam(matrixW1Id, 0, index));
    // IntFloatVector row = (IntFloatVector) ((GetRowResult) client1.get(func)).getRow();
    IntFloatVector row = (IntFloatVector) client1.get(0, index);
    for (int id : index) {
        // System.out.println("id=" + id + ", value=" + row.get(id));
        assertEquals(row.get(id), deltaVec.get(id), 0.000001);
    }
    Assert.assertTrue(index.length == row.size());
}

Example 15 with MatrixClient

use of com.tencent.angel.psagent.matrix.MatrixClient in project angel by Tencent.

the class IndexGetRowsHashTest method testSparseFloatLongKeyUDF.

public void testSparseFloatLongKeyUDF() throws Exception {
    Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
    MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_FLOAT_LONG_MAT, 0);
    int matrixW1Id = client1.getMatrixId();
    long[] index = genLongIndexs(feaNum, nnz);
    LongFloatVector deltaVec = null;
    for (int rowId = 0; rowId < rowNum; rowId++) {
        deltaVec = new LongFloatVector(feaNum, new LongFloatSparseVectorStorage(feaNum));
        for (int i = 0; i < index.length; i++) {
            deltaVec.set(index[i], index[i]);
        }
        client1.increment(rowId, deltaVec, true);
    }
    int[] rowIds = new int[rowNum];
    for (int i = 0; i < rowNum; i++) {
        rowIds[i] = i;
    }
    Vector[] rows = client1.get(rowIds, index);
    for (int i = 0; i < rowNum; i++) {
        for (long id : index) {
            Assert.assertEquals(((LongFloatVector) rows[i]).get(id), deltaVec.get(id), zero);
        }
    }
}