Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 21 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MatrixClientAdapter method dispatchGetRows.

/**
 * Split the rowIndex to batches to generate rpc dispatcher items
 *
 * @param rowIndex rowIds needed to been requested from PS
 */
private void dispatchGetRows(RowIndex rowIndex, int rpcBatchSize, int clock) {
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    // Get the partition to sub-row splits map:use to storage the rows stored in a matrix partition
    Map<PartitionKey, List<RowIndex>> partToRowIndexMap = PSAgentContext.get().getMatrixMetaManager().getPartitionToRowIndexMap(rowIndex, rpcBatchSize);
    List<RowIndex> rowIds;
    int size;
    // Generate dispatch items and add them to the corresponding queues
    int totalRequestNumber = 0;
    for (Entry<PartitionKey, List<RowIndex>> entry : partToRowIndexMap.entrySet()) {
        totalRequestNumber += entry.getValue().size();
    }
    GetRowsFlowRequest request = new GetRowsFlowRequest(rowIndex, clock);
    // Filter the rowIds which are fetching now
    ReentrantLock lock = getLock(rowIndex.getMatrixId());
    Int2IntOpenHashMap rowIndexToPartSizeMap;
    try {
        lock.lock();
        rowIndexToPartSizeMap = matrixToRowSplitSizeCache.get(rowIndex.getMatrixId());
    } finally {
        lock.unlock();
    }
    GetRowsFlowCache cache = new GetRowsFlowCache(totalRequestNumber, rowIndex.getMatrixId(), rowIndexToPartSizeMap);
    for (Entry<PartitionKey, List<RowIndex>> entry : partToRowIndexMap.entrySet()) {
        totalRequestNumber += entry.getValue().size();
        rowIds = entry.getValue();
        size = rowIds.size();
        for (int i = 0; i < size; i++) {
            cache.addResult(matrixClient.getRowsSplit(entry.getKey(), rowIndexToList(rowIds.get(i)), clock));
        }
    }
    requestToResponseMap.put(request, cache);
}

Example 22 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class TaskContext method globalSync.

/**
 * Global sync with special matrix,still wait until all matrixes's clock is synchronized.
 *
 * @param matrixId the matrix id
 * @throws InterruptedException
 */
public void globalSync(int matrixId) throws InterruptedException {
    ClockCache clockCache = PSAgentContext.get().getClockCache();
    List<PartitionKey> pkeys = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int syncTimeIntervalMS = PSAgentContext.get().getConf().getInt(AngelConf.ANGEL_PSAGENT_CACHE_SYNC_TIMEINTERVAL_MS, AngelConf.DEFAULT_ANGEL_PSAGENT_CACHE_SYNC_TIMEINTERVAL_MS);
    while (true) {
        boolean sync = true;
        for (PartitionKey pkey : pkeys) {
            if (clockCache.getClock(matrixId, pkey) < getMatrixClock(matrixId)) {
                sync = false;
                break;
            }
        }
        if (!sync) {
            Thread.sleep(syncTimeIntervalMS);
        } else {
            break;
        }
    }
}

Example 23 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class TaskContext method getPSMatrixClock.

/**
 * Get the clock value of a matrix
 * @param matrixId matrix id
 * @return clock value
 */
public int getPSMatrixClock(int matrixId) {
    ClockCache clockCache = PSAgentContext.get().getClockCache();
    List<PartitionKey> pkeys = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int size = pkeys.size();
    int clock = Integer.MAX_VALUE;
    int partClock = 0;
    for (int i = 0; i < size; i++) {
        partClock = clockCache.getClock(matrixId, pkeys.get(i));
        if (partClock < clock) {
            clock = partClock;
        }
    }
    return clock;
}

Example 24 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class LongIndexGetFunc method partitionGet.

/**
 * Each server partition execute this function and return values of specified index.
 * @param partParam the partition parameter
 * @return values of specified index
 */
@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    ServerPartition part = psContext.getMatrixStorageManager().getPart(partParam.getMatrixId(), partParam.getPartKey().getPartitionId());
    if (part != null) {
        int rowId = ((LongIndexPartGetParam) partParam).getRowId();
        PartitionKey partKey = partParam.getPartKey();
        ServerRow row = part.getRow(rowId);
        RowType rowType = row.getRowType();
        switch(rowType) {
            case T_DOUBLE_SPARSE_LONGKEY:
            case T_DOUBLE_SPARSE_LONGKEY_COMPONENT:
                {
                    return new LongIndexGetResult(partKey, ((ServerSparseDoubleLongKeyRow) row).getValues(((LongIndexPartGetParam) partParam).getIndex()));
                }
            default:
                throw new UnsupportedOperationException("Unsupport operation: update " + rowType + " to " + this.getClass().getName());
        }
    }
    return null;
}

Example 25 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class LongIndexGetResult method deserialize.

@Override
public void deserialize(ByteBuf buf) {
    partKey = new PartitionKey();
    partKey.deserialize(buf);
    int len = buf.readInt();
    values = new double[len];
    for (int i = 0; i < len; i++) {
        values[i] = buf.readDouble();
    }
}