Examples with DenseIntVector com.tencent.angel.ml.math.vector.DenseIntVector used on opensource projects

Example 6 with DenseIntVector

use of com.tencent.angel.ml.math.vector.DenseIntVector in project angel by Tencent.

the class MatrixOpLogTest method testUDF.

@Test
public void testUDF() throws ServiceException, IOException, InvalidParameterException, AngelException, InterruptedException, ExecutionException {
    Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
    MatrixClient w1Task0Client = worker.getPSAgent().getMatrixClient("w1", 0);
    MatrixClient w1Task1Client = worker.getPSAgent().getMatrixClient("w1", 1);
    int matrixW1Id = w1Task0Client.getMatrixId();
    List<Integer> rowIndexes = new ArrayList<Integer>();
    for (int i = 0; i < 100; i++) {
        rowIndexes.add(i);
    }
    GetRowsFunc func = new GetRowsFunc(new GetRowsParam(matrixW1Id, rowIndexes));
    int[] delta = new int[100000];
    for (int i = 0; i < 100000; i++) {
        delta[i] = 1;
    }
    // DenseIntVector deltaVec = new DenseIntVector(100000, delta);
    // deltaVec.setMatrixId(matrixW1Id);
    // deltaVec.setRowId(0);
    int index = 0;
    while (index++ < 10) {
        Map<Integer, TVector> rows = ((GetRowsResult) w1Task0Client.get(func)).getRows();
        for (Entry<Integer, TVector> rowEntry : rows.entrySet()) {
            LOG.info("index " + rowEntry.getKey() + " sum of w1 = " + sum((DenseIntVector) rowEntry.getValue()));
        }
        for (int i = 0; i < 100; i++) {
            DenseIntVector deltaVec = new DenseIntVector(100000, delta);
            deltaVec.setMatrixId(matrixW1Id);
            deltaVec.setRowId(i);
            w1Task0Client.increment(deltaVec);
            deltaVec = new DenseIntVector(100000, delta);
            deltaVec.setMatrixId(matrixW1Id);
            deltaVec.setRowId(i);
            w1Task1Client.increment(deltaVec);
        }
        w1Task0Client.clock().get();
        w1Task1Client.clock().get();
    }
}

Example 7 with DenseIntVector

use of com.tencent.angel.ml.math.vector.DenseIntVector in project angel by Tencent.

the class TransportTest method testGetFlowDenseIntMatrix.

@Test
public void testGetFlowDenseIntMatrix() throws Exception {
    try {
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        MatrixClient mat = worker.getPSAgent().getMatrixClient("dense_int_mat_1", 0);
        DenseIntMatrix expect = new DenseIntMatrix(diRow, diCol);
        RowIndex rowIndex = new RowIndex();
        for (int i = 0; i < diRow; i++) rowIndex.addRowId(i);
        GetRowsResult result = mat.getRowsFlow(rowIndex, diRow / 2);
        TVector row;
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
        Random rand = new Random(System.currentTimeMillis());
        for (int rowId = 0; rowId < diRow; rowId++) {
            DenseIntVector update = new DenseIntVector(diCol);
            for (int j = 0; j < ddCol; j += 3) update.set(j, rand.nextInt());
            mat.increment(rowId, update);
            expect.getRow(rowId).plusBy(update);
        }
        mat.clock().get();
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while (true) {
            row = result.poll();
            if (result.isFetchOver() && row == null)
                break;
            if (row == null)
                continue;
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
    } catch (Exception x) {
        LOG.error("run testGetFlowDenseIntMatrix failed ", x);
        throw x;
    }
}

Example 8 with DenseIntVector

use of com.tencent.angel.ml.math.vector.DenseIntVector in project angel by Tencent.

the class PeriodHATest method testHA.

@Test
public void testHA() throws Exception {
    ParameterServerId ps1Id = new ParameterServerId(0);
    final ParameterServerId ps2Id = new ParameterServerId(1);
    PSAttemptId ps1Attempt0Id = new PSAttemptId(ps1Id, 0);
    PSAttemptId ps2Attempt0Id = new PSAttemptId(ps2Id, 0);
    PSAttemptId ps2Attempt1Id = new PSAttemptId(ps2Id, 1);
    ParameterServer ps1Attempt0 = LocalClusterContext.get().getPS(ps1Attempt0Id).getPS();
    ParameterServer ps2Attempt0 = LocalClusterContext.get().getPS(ps2Attempt0Id).getPS();
    WorkerId worker0Id = new WorkerId(new WorkerGroupId(0), 0);
    WorkerAttemptId worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    Worker worker0 = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
    TaskContext task0Context = worker0.getTaskManager().getRunningTask().get(task0Id).getTaskContext();
    MatrixClient matrixClient = task0Context.getMatrix("w1");
    int iterNum = 20;
    for (int i = 0; i < iterNum; i++) {
        DenseIntVector update = new DenseIntVector(dim);
        for (int j = 0; j < dim; j++) {
            update.set(j, 1);
        }
        update.setMatrixId(matrixClient.getMatrixId());
        update.setRowId(0);
        matrixClient.increment(update);
        matrixClient.clock().get();
        Thread.sleep(1000);
        MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
        ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps1w1.getPartition(0));
        assertNotNull(ps1w1.getPartition(1));
        IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
        int part0Size = ps1w1.getRow(0, 0).size();
        IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
        int part1Size = ps1w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
        MatrixStorageManager ps2Storage = ps2Attempt0.getMatrixStorageManager();
        ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps2w1.getPartition(0));
        assertNotNull(ps2w1.getPartition(1));
        row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
        part0Size = ps2w1.getRow(0, 0).size();
        row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
        part1Size = ps2w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
    }
    LOG.info("===================================================================ps2 failed");
    ps2Attempt0.failed("exit");
    for (int i = iterNum; i < 2 * iterNum; i++) {
        DenseIntVector update = new DenseIntVector(dim);
        for (int j = 0; j < dim; j++) {
            update.set(j, 1);
        }
        update.setMatrixId(matrixClient.getMatrixId());
        update.setRowId(0);
        matrixClient.increment(update);
        matrixClient.clock().get();
        Thread.sleep(1000);
        MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
        ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps1w1.getPartition(0));
        assertNotNull(ps1w1.getPartition(1));
        IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
        int part0Size = ps1w1.getRow(0, 0).size();
        IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
        int part1Size = ps1w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
    }
    ParameterServer ps2Attempt = LocalClusterContext.get().getPS(ps2Attempt1Id).getPS();
    for (int i = iterNum * 2; i < 3 * iterNum; i++) {
        DenseIntVector update = new DenseIntVector(dim);
        for (int j = 0; j < dim; j++) {
            update.set(j, 1);
        }
        update.setMatrixId(matrixClient.getMatrixId());
        update.setRowId(0);
        matrixClient.increment(update);
        matrixClient.clock().get();
        Thread.sleep(1000);
        MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
        ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps1w1.getPartition(0));
        assertNotNull(ps1w1.getPartition(1));
        IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
        int part0Size = ps1w1.getRow(0, 0).size();
        IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
        int part1Size = ps1w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
        MatrixStorageManager ps2Storage = ps2Attempt.getMatrixStorageManager();
        ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
        assertNotNull(ps2w1.getPartition(0));
        assertNotNull(ps2w1.getPartition(1));
        row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
        part0Size = ps2w1.getRow(0, 0).size();
        row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
        part1Size = ps2w1.getRow(1, 0).size();
        assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
        assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
    }
}

Example 9 with DenseIntVector

use of com.tencent.angel.ml.math.vector.DenseIntVector in project angel by Tencent.

the class DenseIntMatrixTest method plusBy3.

@Test
public void plusBy3() throws Exception {
    int[][] value = { { 1, 2 }, { 3, 4 } };
    DenseIntMatrix mat = new DenseIntMatrix(2, 2, value);
    TIntVector vec = new DenseIntVector(2, new int[] { 1, 2 });
    vec.setRowId(0);
    mat.plusBy(vec);
    assertEquals(2, mat.get(0, 0));
    assertEquals(4, mat.get(0, 1));
    assertEquals(3, mat.get(1, 0));
    assertEquals(4, mat.get(1, 1));
    DenseIntMatrix mat_1 = new DenseIntMatrix(2, 2);
    mat_1.plusBy(vec);
    assertEquals(1, mat_1.get(0, 0));
    assertEquals(2, mat_1.get(0, 1));
    assertEquals(0, mat_1.get(1, 0));
    assertEquals(0, mat_1.get(1, 1));
}

Example 10 with DenseIntVector

use of com.tencent.angel.ml.math.vector.DenseIntVector in project angel by Tencent.

the class DenseIntMatrixTest method nonZeroNum.

@Test
public void nonZeroNum() throws Exception {
    DenseIntMatrix mat = new DenseIntMatrix(2, 2, new int[][] { { 0, 2 }, { 0, 4 } });
    assertEquals(2, mat.nonZeroNum());
    DenseIntMatrix mat_1 = new DenseIntMatrix(2, 2);
    DenseIntVector vec = new DenseIntVector(2, new int[] { 1, 2 });
    vec.setRowId(0);
    mat_1.plusBy(vec);
    assertEquals(2, mat_1.nonZeroNum());
}