Examples with TVector com.tencent.angel.ml.math.TVector used on opensource projects

Example 11 with TVector

use of com.tencent.angel.ml.math.TVector in project angel by Tencent.

the class RowSplitCombineUtils method combineServerDenseDoubleRowSplits.

private static TVector combineServerDenseDoubleRowSplits(List<ServerRow> rowSplits, MatrixMeta matrixMeta, int rowIndex) {
    int colNum = (int) matrixMeta.getColNum();
    double[] dataArray = new double[colNum];
    Collections.sort(rowSplits, serverRowComp);
    int clock = Integer.MAX_VALUE;
    int size = rowSplits.size();
    for (int i = 0; i < size; i++) {
        if (rowSplits.get(i) == null) {
            continue;
        }
        if (rowSplits.get(i).getClock() < clock) {
            clock = rowSplits.get(i).getClock();
        }
        ((ServerDenseDoubleRow) rowSplits.get(i)).mergeTo(dataArray);
    }
    TVector row = new DenseDoubleVector(colNum, dataArray);
    row.setMatrixId(matrixMeta.getId());
    row.setRowId(rowIndex);
    row.setClock(clock);
    return row;
}

Example 12 with TVector

use of com.tencent.angel.ml.math.TVector in project angel by Tencent.

the class MatrixOpLogTest method testUDF.

@Test
public void testUDF() throws ServiceException, IOException, InvalidParameterException, AngelException, InterruptedException, ExecutionException {
    Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
    MatrixClient w1Task0Client = worker.getPSAgent().getMatrixClient("w1", 0);
    MatrixClient w1Task1Client = worker.getPSAgent().getMatrixClient("w1", 1);
    int matrixW1Id = w1Task0Client.getMatrixId();
    List<Integer> rowIndexes = new ArrayList<Integer>();
    for (int i = 0; i < 100; i++) {
        rowIndexes.add(i);
    }
    GetRowsFunc func = new GetRowsFunc(new GetRowsParam(matrixW1Id, rowIndexes));
    int[] delta = new int[100000];
    for (int i = 0; i < 100000; i++) {
        delta[i] = 1;
    }
    // DenseIntVector deltaVec = new DenseIntVector(100000, delta);
    // deltaVec.setMatrixId(matrixW1Id);
    // deltaVec.setRowId(0);
    int index = 0;
    while (index++ < 10) {
        Map<Integer, TVector> rows = ((GetRowsResult) w1Task0Client.get(func)).getRows();
        for (Entry<Integer, TVector> rowEntry : rows.entrySet()) {
            LOG.info("index " + rowEntry.getKey() + " sum of w1 = " + sum((DenseIntVector) rowEntry.getValue()));
        }
        for (int i = 0; i < 100; i++) {
            DenseIntVector deltaVec = new DenseIntVector(100000, delta);
            deltaVec.setMatrixId(matrixW1Id);
            deltaVec.setRowId(i);
            w1Task0Client.increment(deltaVec);
            deltaVec = new DenseIntVector(100000, delta);
            deltaVec.setMatrixId(matrixW1Id);
            deltaVec.setRowId(i);
            w1Task1Client.increment(deltaVec);
        }
        w1Task0Client.clock().get();
        w1Task1Client.clock().get();
    }
}

Example 13 with TVector

use of com.tencent.angel.ml.math.TVector in project angel by Tencent.

the class TransportTest method testGetFlowDenseIntMatrix.

@Test
public void testGetFlowDenseIntMatrix() throws Exception {
    try {
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        MatrixClient mat = worker.getPSAgent().getMatrixClient("dense_int_mat_1", 0);
        DenseIntMatrix expect = new DenseIntMatrix(diRow, diCol);
        RowIndex rowIndex = new RowIndex();
        for (int i = 0; i < diRow; i++) rowIndex.addRowId(i);
        GetRowsResult result = mat.getRowsFlow(rowIndex, diRow / 2);
        TVector row;
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
        Random rand = new Random(System.currentTimeMillis());
        for (int rowId = 0; rowId < diRow; rowId++) {
            DenseIntVector update = new DenseIntVector(diCol);
            for (int j = 0; j < ddCol; j += 3) update.set(j, rand.nextInt());
            mat.increment(rowId, update);
            expect.getRow(rowId).plusBy(update);
        }
        mat.clock().get();
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while (true) {
            row = result.poll();
            if (result.isFetchOver() && row == null)
                break;
            if (row == null)
                continue;
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
    } catch (Exception x) {
        LOG.error("run testGetFlowDenseIntMatrix failed ", x);
        throw x;
    }
}

Example 14 with TVector

use of com.tencent.angel.ml.math.TVector in project angel by Tencent.

the class MatrixClientAdapter method getRow.

/**
 * Get a matrix row from parameter servers
 *
 * @param matrixId matrix id
 * @param rowIndex row index
 * @param clock    clock value
 * @return TVector matrix row
 * @throws ExecutionException   exception thrown when attempting to retrieve the result of a task
 *                              that aborted by throwing an exception
 * @throws InterruptedException interrupted while wait the result
 */
public TVector getRow(int matrixId, int rowIndex, int clock) throws InterruptedException, ExecutionException {
    LOG.debug("start to getRow request, matrix=" + matrixId + ", rowIndex=" + rowIndex + ", clock=" + clock);
    long startTs = System.currentTimeMillis();
    // Wait until the clock value of this row is greater than or equal to the value
    PSAgentContext.get().getConsistencyController().waitForClock(matrixId, rowIndex, clock);
    LOG.debug("getRow wait clock time=" + (System.currentTimeMillis() - startTs));
    startTs = System.currentTimeMillis();
    // Get partitions for this row
    List<PartitionKey> partList = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId, rowIndex);
    GetRowRequest request = new GetRowRequest(matrixId, rowIndex, clock);
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    GetRowPipelineCache responseCache = (GetRowPipelineCache) requestToResponseMap.get(request);
    if (responseCache == null) {
        responseCache = new GetRowPipelineCache(partList.size(), meta.getRowType());
        GetRowPipelineCache oldCache = (GetRowPipelineCache) requestToResponseMap.putIfAbsent(request, responseCache);
        if (oldCache != null) {
            responseCache = oldCache;
        }
    }
    // First get this row from matrix storage
    MatrixStorage matrixStorage = PSAgentContext.get().getMatrixStorageManager().getMatrixStoage(matrixId);
    try {
        responseCache.getDistinctLock().lock();
        // If the row exists in the matrix storage and the clock value meets the requirements, just
        // return
        TVector row = matrixStorage.getRow(rowIndex);
        if (row != null && row.getClock() >= clock) {
            return row;
        }
        // Get row splits of this row from the matrix cache first
        MatricesCache matricesCache = PSAgentContext.get().getMatricesCache();
        MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
        int size = partList.size();
        for (int i = 0; i < size; i++) {
            ServerRow rowSplit = matricesCache.getRowSplit(matrixId, partList.get(i), rowIndex);
            if (rowSplit != null && rowSplit.getClock() >= clock) {
                responseCache.addRowSplit(rowSplit);
            } else {
                // If the row split does not exist in cache, get it from parameter server
                responseCache.addRowSplit(matrixClient.getRowSplit(partList.get(i), rowIndex, clock));
            }
        }
        // Wait the final result
        row = responseCache.getMergedResult().get();
        LOG.debug("get row use time=" + (System.currentTimeMillis() - startTs));
        // Put it to the matrix cache
        matrixStorage.addRow(rowIndex, row);
        return row;
    } finally {
        responseCache.getDistinctLock().unlock();
        requestToResponseMap.remove(request);
    }
}

Example 15 with TVector

use of com.tencent.angel.ml.math.TVector in project angel by Tencent.

the class RowSplitCombineUtils method combineServerSparseIntRowSplits.

private static TVector combineServerSparseIntRowSplits(List<ServerRow> rowSplits, MatrixMeta matrixMeta, int rowIndex) {
    int colNum = (int) matrixMeta.getColNum();
    int splitNum = rowSplits.size();
    int totalElemNum = 0;
    int[] lens = new int[splitNum];
    Collections.sort(rowSplits, serverRowComp);
    int elemNum = 0;
    for (int i = 0; i < splitNum; i++) {
        elemNum = rowSplits.get(i).size();
        totalElemNum += elemNum;
        lens[i] = elemNum;
    }
    int[] indexes = new int[totalElemNum];
    int[] values = new int[totalElemNum];
    int clock = Integer.MAX_VALUE;
    int startPos = 0;
    for (int i = 0; i < splitNum; i++) {
        if (rowSplits.get(i).getClock() < clock) {
            clock = rowSplits.get(i).getClock();
        }
        ((ServerSparseIntRow) rowSplits.get(i)).mergeTo(indexes, values, startPos, lens[i]);
        startPos += lens[i];
    }
    TVector row = new SparseIntVector(colNum, indexes, values);
    row.setMatrixId(matrixMeta.getId());
    row.setRowId(rowIndex);
    row.setClock(clock);
    return row;
}