Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 6 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class WorkerPool method getRowSplit.

/**
 * Get a row split
 * @param seqId rpc request id
 * @param request rpc request
 * @return serialized rpc response contains the row split
 */
private ByteBuf getRowSplit(int seqId, GetRowSplitRequest request) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("get row request=" + request + " with seqId=" + seqId);
    }
    PartitionKey partKey = request.getPartKey();
    int clock = request.getClock();
    long startTs = System.currentTimeMillis();
    GetRowSplitResponse response = new GetRowSplitResponse();
    if (!isClockReady(partKey, clock)) {
        response.setResponseType(ResponseType.CLOCK_NOTREADY);
    } else {
        ServerRow row = context.getMatrixStorageManager().getRow(partKey.getMatrixId(), request.getRowIndex(), partKey.getPartitionId());
        row.setClock(context.getClockVectorManager().getPartClock(partKey.getMatrixId(), partKey.getPartitionId()));
        response.setResponseType(ResponseType.SUCCESS);
        response.setRowSplit(row);
    }
    ByteBuf buf = ByteBufUtils.newByteBuf(4 + response.bufferLen(), useDirectorBuffer);
    buf.writeInt(seqId);
    response.serialize(buf);
    if (LOG.isDebugEnabled()) {
        LOG.debug("response for request " + request + " serialize use time=" + (System.currentTimeMillis() - startTs) + ", response buf=" + buf);
    }
    return buf;
}

Example 7 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class PSAgentMatrixMetaManager method getPartitionsFromMeta.

private List<PartitionKey> getPartitionsFromMeta(int matrixId, int rowIndex) {
    List<PartitionKey> partitionKeys = new ArrayList<>();
    Iterator<PartitionMeta> iter = matrixMetaManager.getMatrixMeta(matrixId).getPartitionMetas().values().iterator();
    while (iter.hasNext()) {
        PartitionKey partitionKey = iter.next().getPartitionKey();
        if (partitionKey.getMatrixId() == matrixId && partitionKey.getStartRow() <= rowIndex && partitionKey.getEndRow() > rowIndex)
            partitionKeys.add(partitionKey);
    }
    partitionKeys.sort(new Comparator<PartitionKey>() {

        @Override
        public int compare(PartitionKey p1, PartitionKey p2) {
            return 0;
        }
    });
    return partitionKeys;
}

Example 8 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ServerPartitionTest method testDeserialize.

@Test
public void testDeserialize() throws Exception {
    ByteBuf buf = Unpooled.buffer(16);
    serverPartition.serialize(buf);
    PartitionKey partitionKeyNew = new PartitionKey(2, 1, 1, 2, 8, 10);
    ServerPartition serverPartitionNew = new ServerPartition(partitionKeyNew, RowType.T_DOUBLE_DENSE);
    assertNotEquals(serverPartition.getPartitionKey().getPartitionId(), serverPartitionNew.getPartitionKey().getPartitionId());
    serverPartitionNew.deserialize(buf);
    assertEquals(serverPartition.getPartitionKey().getPartitionId(), serverPartitionNew.getPartitionKey().getPartitionId());
}

Example 9 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class VectorSplitTest method sparseSortedVectorSplit.

@Test
public void sparseSortedVectorSplit() {
    int[] offsets = { 0, 2, 4, 6, 8 };
    double[] values = { 0.0, 2.0, 4.0, 6.0, 8.0 };
    SparseDoubleSortedVector vector = new SparseDoubleSortedVector(10, offsets, values);
    vector.setRowId(0);
    PartitionKey key1 = new PartitionKey(0, 0, 0, 0, 1, 5);
    PartitionKey key2 = new PartitionKey(1, 0, 0, 5, 1, 10);
    List<PartitionKey> keys = new ArrayList<>();
    keys.add(key1);
    keys.add(key2);
    HashMap<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, keys);
    Assert.assertEquals(keys.size(), splits.size());
    int[] offset1 = { 0, 2, 4 };
    double[] values1 = { 0.0, 2.0, 4.0 };
    SparseDoubleRowUpdateSplit split1 = (SparseDoubleRowUpdateSplit) splits.get(key1);
    Assert.assertNotNull(split1);
    Assert.assertEquals(offset1.length, split1.size());
    Assert.assertEquals(0, split1.getStart());
    Assert.assertEquals(3, split1.getEnd());
    Assert.assertArrayEquals(offset1, Arrays.copyOfRange(split1.getOffsets(), (int) split1.getStart(), (int) split1.getEnd()));
    for (int i = 0; i < split1.size(); i++) {
        Assert.assertEquals(values1[i], split1.getValues()[(int) split1.getStart() + i], 0.0);
    }
    int[] offset2 = { 6, 8 };
    double[] values2 = { 6.0, 8.0 };
    SparseDoubleRowUpdateSplit split2 = (SparseDoubleRowUpdateSplit) splits.get(key2);
    Assert.assertNotNull(split2);
    Assert.assertEquals(offset2.length, split2.size());
    Assert.assertEquals(3, split2.getStart());
    Assert.assertEquals(5, split2.getEnd());
    Assert.assertArrayEquals(offset2, Arrays.copyOfRange(split2.getOffsets(), (int) split2.getStart(), (int) split2.getEnd()));
    for (int i = 0; i < split2.size(); i++) {
        Assert.assertEquals(values2[i], split2.getValues()[(int) split2.getStart() + i], 0.0);
    }
    LOG.info("Pass sparseSortedVectorSplit Test");
}

Example 10 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ArrayAggrParam method split.

@Override
public List<PartitionGetParam> split() {
    List<PartitionKey> parts = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int size = parts.size();
    List<PartitionGetParam> partParams = new ArrayList<PartitionGetParam>(size);
    for (PartitionKey part : parts) {
        if (Utils.withinPart(part, new int[] { rowId })) {
            long startCol = part.getStartCol();
            long endCol = part.getEndCol();
            ArrayList<Long> partCols = new ArrayList<>();
            for (long col : this.cols) {
                if (col >= startCol && col < endCol) {
                    partCols.add(col);
                }
            }
            if (partCols.size() > 0) {
                partParams.add(new ArrayPartitionAggrParam(matrixId, part, rowId, Utils.longListToArray(partCols)));
            }
        }
    }
    return partParams;
}