Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 66 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseDoubleCompVector.

public static CompSparseDoubleVector mergeSparseDoubleCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int size = partKeys.size();
    SparseDoubleVector[] splitVecs = new SparseDoubleVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            splitVecs[i] = new SparseDoubleVector((int) meta.getColNum(), param.getPartKeyToIndexesMap().get(partKeys.get(i)), ((IndexPartGetDoubleResult) partKeyToResultMap.get(partKeys.get(i))).getValues());
        }
    }
    return new CompSparseDoubleVector(meta.getId(), param.getRowId(), (int) meta.getColNum(), partKeys.toArray(new PartitionKey[0]), splitVecs);
}

Example 67 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseIntCompVector.

public static CompSparseIntVector mergeSparseIntCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int size = partKeys.size();
    SparseIntVector[] splitVecs = new SparseIntVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            splitVecs[i] = new SparseIntVector((int) meta.getColNum(), param.getPartKeyToIndexesMap().get(partKeys.get(i)), ((IndexPartGetIntResult) partKeyToResultMap.get(partKeys.get(i))).getValues());
        }
    }
    return new CompSparseIntVector(meta.getId(), param.getRowId(), (int) meta.getColNum(), partKeys.toArray(new PartitionKey[0]), splitVecs);
}

Example 68 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class PartitionUpdateParam method deserialize.

@Override
public void deserialize(ByteBuf buf) {
    matrixId = buf.readInt();
    updateClock = buf.readBoolean();
    if (buf.isReadable()) {
        if (partKey == null) {
            partKey = new PartitionKey();
        }
        partKey.deserialize(buf);
    }
}

Example 69 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class AsyncEventPusher method put.

@Override
public void put(PartitionRequest request, ByteBuf msg, PartitionLocation partLoc) {
    request.setComeFromPs(true);
    msg.resetReaderIndex();
    msg.setBoolean(8, true);
    PartitionKey partKey = request.getPartKey();
    if (partLoc.psLocs.size() == 1) {
        return;
    } else {
        if (partLoc.psLocs.get(0).psId.equals(context.getPSAttemptId().getPsId())) {
            int size = partLoc.psLocs.size();
            for (int i = 1; i < size; i++) {
                resultMap.put(new UpdateKey(idGen.incrementAndGet(), partKey, partLoc.psLocs.get(i)), psClient.put(partLoc.psLocs.get(i).psId, partLoc.psLocs.get(i).loc, request, msg.copy()));
            }
            msg.release();
        }
    }
}

Example 70 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class PeriodPusher method start.

/**
 * Start
 */
public void start() {
    super.start();
    dispatcher = new Thread(() -> {
        ParameterServerId psId = context.getPSAttemptId().getPsId();
        while (!stopped.get() && !Thread.interrupted()) {
            try {
                Thread.sleep(pushIntervalMs);
                Map<PartitionKey, Integer> parts = getAndClearAllNeedRecoverParts();
                Map<RecoverPartKey, FutureResult> futures = new HashMap<>(parts.size());
                for (PartitionKey part : parts.keySet()) {
                    PartitionLocation partLoc = context.getMaster().getPartLocation(part.getMatrixId(), part.getPartitionId());
                    if ((partLoc.psLocs.size() > 1) && psId.equals(partLoc.psLocs.get(0).psId)) {
                        int size = partLoc.psLocs.size();
                        for (int i = 1; i < size; i++) {
                            RecoverPartKey partKey = new RecoverPartKey(part, partLoc.psLocs.get(i));
                            LOG.info("Start to backup partition " + partKey.partKey + " to " + partKey.psLoc);
                            futures.put(partKey, recover(partKey));
                        }
                    }
                }
                waitResults(futures);
            } catch (Exception e) {
                if (!stopped.get()) {
                    LOG.error("recover parts failed ", e);
                }
            }
        }
    });
    dispatcher.setName("psha-push-dispatcher");
    dispatcher.start();
}