Examples with RowIndex com.tencent.angel.psagent.matrix.transport.adapter.RowIndex used on opensource projects

Example 1 with RowIndex

use of com.tencent.angel.psagent.matrix.transport.adapter.RowIndex in project angel by Tencent.

the class PSAgentMatrixMetaManager method getPartitionToRowIndexMap.

/**
 * Get the partitions the rows in.
 *
 * @param rowIndex matrix id and row indexes
 * @param batchNumber the split batch size.
 * @return Map<PartitionKey, List<Integer>> partitions to the rows contained in the partition map
 */
public Map<PartitionKey, List<RowIndex>> getPartitionToRowIndexMap(RowIndex rowIndex, int batchNumber) {
    Map<PartitionKey, List<RowIndex>> partToRowIndexMap = new TreeMap<PartitionKey, List<RowIndex>>();
    if (rowIndex.getRowIds() == null) {
        return partToRowIndexMap;
    }
    IntOpenHashSet rowIdSet = rowIndex.getRowIds();
    IntOpenHashSet filtedRowIdSet = rowIndex.getFiltedIdSet();
    int[] rowIds = new int[rowIdSet.size() - filtedRowIdSet.size()];
    int count = 0;
    for (int rowId : rowIdSet) {
        if (!filtedRowIdSet.contains(rowId)) {
            rowIds[count++] = rowId;
        }
    }
    Arrays.sort(rowIds);
    int partNum = 0;
    for (int i = 0; i < rowIds.length; i++) {
        List<PartitionKey> partKeys = getPartitions(rowIndex.getMatrixId(), rowIds[i]);
        partNum = partKeys.size();
        for (int j = 0; j < partNum; j++) {
            PartitionKey partitionKey = partKeys.get(j);
            List<RowIndex> indexList = partToRowIndexMap.get(partitionKey);
            if (indexList == null) {
                indexList = new ArrayList<RowIndex>();
                partToRowIndexMap.put(partitionKey, indexList);
                indexList.add(new RowIndex(rowIndex.getMatrixId(), rowIndex));
            }
            RowIndex index = indexList.get(indexList.size() - 1);
            if (index.getRowsNumber() >= batchNumber) {
                index = new RowIndex(rowIndex.getMatrixId(), rowIndex);
                indexList.add(index);
            }
            index.addRowId(rowIds[i]);
        }
    }
    return partToRowIndexMap;
}

Example 2 with RowIndex

use of com.tencent.angel.psagent.matrix.transport.adapter.RowIndex in project angel by Tencent.

the class TransportTest method testGetFlowDenseDoubleMatrix.

@Test
public void testGetFlowDenseDoubleMatrix() throws Exception {
    try {
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        MatrixClient mat = worker.getPSAgent().getMatrixClient("dense_double_mat_1", 0);
        double[][] data = new double[ddRow][ddCol];
        DenseDoubleMatrix expect = new DenseDoubleMatrix(ddRow, ddCol, data);
        RowIndex rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        GetRowsResult result = mat.getRowsFlow(rowIndex, ddRow / 2);
        TVector row;
        while ((row = result.take()) != null) {
            LOG.info("===========get row index=" + row.getRowId());
            assertArrayEquals(((DenseDoubleVector) expect.getRow(row.getRowId())).getValues(), ((DenseDoubleVector) row).getValues(), 0.0);
        }
        Random rand = new Random(System.currentTimeMillis());
        for (int rowId = 0; rowId < ddRow; rowId++) {
            DenseDoubleVector update = new DenseDoubleVector(ddCol);
            for (int j = 0; j < ddCol; j += 3) update.set(j, rand.nextDouble());
            mat.increment(rowId, update);
            expect.getRow(rowId).plusBy(update);
        }
        mat.clock().get();
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseDoubleVector) expect.getRow(row.getRowId())).getValues(), ((DenseDoubleVector) row).getValues(), 0.0);
        }
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while (true) {
            row = result.poll();
            if (result.isFetchOver() && row == null)
                break;
            if (row == null)
                continue;
            assertArrayEquals(((DenseDoubleVector) expect.getRow(row.getRowId())).getValues(), ((DenseDoubleVector) row).getValues(), 0.0);
        }
    } catch (Exception x) {
        LOG.error("run testGetFlowDenseDoubleMatrix failed ", x);
        throw x;
    }
}

Example 3 with RowIndex

use of com.tencent.angel.psagent.matrix.transport.adapter.RowIndex in project angel by Tencent.

the class TransportTest method testGetFlowDenseIntMatrix.

@Test
public void testGetFlowDenseIntMatrix() throws Exception {
    try {
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        MatrixClient mat = worker.getPSAgent().getMatrixClient("dense_int_mat_1", 0);
        DenseIntMatrix expect = new DenseIntMatrix(diRow, diCol);
        RowIndex rowIndex = new RowIndex();
        for (int i = 0; i < diRow; i++) rowIndex.addRowId(i);
        GetRowsResult result = mat.getRowsFlow(rowIndex, diRow / 2);
        TVector row;
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
        Random rand = new Random(System.currentTimeMillis());
        for (int rowId = 0; rowId < diRow; rowId++) {
            DenseIntVector update = new DenseIntVector(diCol);
            for (int j = 0; j < ddCol; j += 3) update.set(j, rand.nextInt());
            mat.increment(rowId, update);
            expect.getRow(rowId).plusBy(update);
        }
        mat.clock().get();
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while (true) {
            row = result.poll();
            if (result.isFetchOver() && row == null)
                break;
            if (row == null)
                continue;
            assertArrayEquals(((DenseIntVector) expect.getRow(row.getRowId())).getValues(), ((DenseIntVector) row).getValues());
        }
    } catch (Exception x) {
        LOG.error("run testGetFlowDenseIntMatrix failed ", x);
        throw x;
    }
}