Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 56 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static HashMap<PartitionKey, RowUpdateSplit> split(int rowId, int[] values, List<PartitionKey> partitionInfos) {
    HashMap<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    for (PartitionKey partitionKey : partitionInfos) {
        if (rowId >= partitionKey.getStartRow() && rowId < partitionKey.getEndRow()) {
            RowUpdateSplit split = new DenseIntRowUpdateSplit(rowId, (int) partitionKey.getStartCol(), (int) partitionKey.getEndCol(), values);
            ret.put(partitionKey, split);
        }
    }
    return ret;
}

Example 57 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static HashMap<PartitionKey, RowUpdateSplit> split(int rowId, float[] values, List<PartitionKey> partitionInfos) {
    HashMap<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    for (PartitionKey partitionKey : partitionInfos) {
        if (rowId >= partitionKey.getStartRow() && rowId < partitionKey.getEndRow()) {
            RowUpdateSplit split = new DenseFloatRowUpdateSplit(rowId, (int) partitionKey.getStartCol(), (int) partitionKey.getEndCol(), values);
            ret.put(partitionKey, split);
        }
    }
    return ret;
}

Example 58 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class RowUpdateSplitUtils method split.

public static HashMap<PartitionKey, RowUpdateSplit> split(int rowId, double[] values, List<PartitionKey> partitionInfos) {
    HashMap<PartitionKey, RowUpdateSplit> ret = new HashMap<>();
    for (PartitionKey partitionKey : partitionInfos) {
        if (rowId >= partitionKey.getStartRow() && rowId < partitionKey.getEndRow()) {
            RowUpdateSplit split = new DenseDoubleRowUpdateSplit(rowId, (int) partitionKey.getStartCol(), (int) partitionKey.getEndCol(), values);
            ret.put(partitionKey, split);
        }
    }
    return ret;
}

Example 59 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MatrixClientAdapter method flush.

/**
 * Flush the matrix oplog to parameter servers.
 *
 * @param matrixId    matrix id
 * @param taskContext task context
 * @param matrixOpLog matrix oplog
 * @param updateClock true means we should update the clock value after update matrix
 * @return Future<VoidResult> flush future result
 */
public Future<VoidResult> flush(int matrixId, TaskContext taskContext, MatrixOpLog matrixOpLog, boolean updateClock) {
    if (!updateClock && (matrixOpLog == null)) {
        FutureResult<VoidResult> ret = new FutureResult<VoidResult>();
        ret.set(new VoidResult(ResponseType.SUCCESS));
        return ret;
    }
    Map<PartitionKey, List<RowUpdateSplit>> psUpdateData = new HashMap<PartitionKey, List<RowUpdateSplit>>();
    FlushRequest request = new FlushRequest(taskContext.getMatrixClock(matrixId), taskContext.getIndex(), matrixId, matrixOpLog, updateClock);
    LOG.debug("start to flush update for matrix=" + matrixId + ", taskIndex=" + taskContext.getIndex());
    long startTs = System.currentTimeMillis();
    // Split the matrix oplog according to the matrix partitions
    if (matrixOpLog != null) {
        matrixOpLog.split(psUpdateData);
    }
    LOG.debug("split use time=" + (System.currentTimeMillis() - startTs));
    // If need update clock, we should send requests to all partitions
    if (updateClock) {
        fillPartRequestForClock(matrixId, psUpdateData, taskContext);
    }
    FlushResponseCache cache = new FlushResponseCache(psUpdateData.size());
    pushUpdates(matrixId, psUpdateData, taskContext, updateClock, cache);
    requestToResponseMap.put(request, cache);
    return cache.getMergedResult();
}

Example 60 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MatrixClientAdapter method getRow.

/**
 * Get a matrix row from parameter servers
 *
 * @param matrixId matrix id
 * @param rowIndex row index
 * @param clock    clock value
 * @return TVector matrix row
 * @throws ExecutionException   exception thrown when attempting to retrieve the result of a task
 *                              that aborted by throwing an exception
 * @throws InterruptedException interrupted while wait the result
 */
public TVector getRow(int matrixId, int rowIndex, int clock) throws InterruptedException, ExecutionException {
    LOG.debug("start to getRow request, matrix=" + matrixId + ", rowIndex=" + rowIndex + ", clock=" + clock);
    long startTs = System.currentTimeMillis();
    // Wait until the clock value of this row is greater than or equal to the value
    PSAgentContext.get().getConsistencyController().waitForClock(matrixId, rowIndex, clock);
    LOG.debug("getRow wait clock time=" + (System.currentTimeMillis() - startTs));
    startTs = System.currentTimeMillis();
    // Get partitions for this row
    List<PartitionKey> partList = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId, rowIndex);
    GetRowRequest request = new GetRowRequest(matrixId, rowIndex, clock);
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    GetRowPipelineCache responseCache = (GetRowPipelineCache) requestToResponseMap.get(request);
    if (responseCache == null) {
        responseCache = new GetRowPipelineCache(partList.size(), meta.getRowType());
        GetRowPipelineCache oldCache = (GetRowPipelineCache) requestToResponseMap.putIfAbsent(request, responseCache);
        if (oldCache != null) {
            responseCache = oldCache;
        }
    }
    // First get this row from matrix storage
    MatrixStorage matrixStorage = PSAgentContext.get().getMatrixStorageManager().getMatrixStoage(matrixId);
    try {
        responseCache.getDistinctLock().lock();
        // If the row exists in the matrix storage and the clock value meets the requirements, just
        // return
        TVector row = matrixStorage.getRow(rowIndex);
        if (row != null && row.getClock() >= clock) {
            return row;
        }
        // Get row splits of this row from the matrix cache first
        MatricesCache matricesCache = PSAgentContext.get().getMatricesCache();
        MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
        int size = partList.size();
        for (int i = 0; i < size; i++) {
            ServerRow rowSplit = matricesCache.getRowSplit(matrixId, partList.get(i), rowIndex);
            if (rowSplit != null && rowSplit.getClock() >= clock) {
                responseCache.addRowSplit(rowSplit);
            } else {
                // If the row split does not exist in cache, get it from parameter server
                responseCache.addRowSplit(matrixClient.getRowSplit(partList.get(i), rowIndex, clock));
            }
        }
        // Wait the final result
        row = responseCache.getMergedResult().get();
        LOG.debug("get row use time=" + (System.currentTimeMillis() - startTs));
        // Put it to the matrix cache
        matrixStorage.addRow(rowIndex, row);
        return row;
    } finally {
        responseCache.getDistinctLock().unlock();
        requestToResponseMap.remove(request);
    }
}