Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 36 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ServerPartitionTest method setUp.

@Before
public void setUp() throws Exception {
    partitionId = 1;
    matrixId = 1;
    startRow = 2;
    startCol = 2;
    endRow = 8;
    endCol = 10;
    rowType = RowType.T_DOUBLE_DENSE;
    partitionKey = new PartitionKey(partitionId, matrixId, startRow, startCol, endRow, endCol);
    serverPartition = new ServerPartition(partitionKey, rowType);
    serverPartition.init();
}

Example 37 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class PSAgentTest method testMatrixLocationManager.

@Test
public void testMatrixLocationManager() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        PSAgentMatrixMetaManager matrixPartitionRouter = psAgent.getMatrixMetaManager();
        PSAgentLocationManager locationCache = psAgent.getLocationManager();
        assertTrue(matrixPartitionRouter != null);
        // test ps location
        Location psLoc = locationCache.getPsLocation(psId);
        String ipRegex = "(2[5][0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})";
        Pattern pattern = Pattern.compile(ipRegex);
        Matcher matcher = pattern.matcher(psLoc.getIp());
        assertTrue(matcher.matches());
        assertTrue(psLoc.getPort() >= 1 && psLoc.getPort() <= 65535);
        int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
        int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
        // test partitions
        List<PartitionKey> partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id);
        assertEquals(partition1Keys.size(), 2);
        List<PartitionKey> partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id);
        assertEquals(partition2Keys.size(), 2);
        partition1Keys.clear();
        partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
        assertEquals(partition1Keys.size(), 2);
        partition2Keys.clear();
        partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
        assertEquals(partition2Keys.size(), 2);
        int rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
        assertEquals(rowPartSize, 2);
        rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
        assertEquals(rowPartSize, 2);
    } catch (Exception x) {
        LOG.error("run testMatrixLocationManager failed ", x);
        throw x;
    }
}

Example 38 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class VectorSplitTest method sparseIntVectorSplit.

@Test
public void sparseIntVectorSplit() {
    int[] offsets = { 0, 2, 4, 6, 8 };
    int[] values = { 0, 2, 4, 6, 8 };
    SparseIntVector vector = new SparseIntVector(10, offsets, values);
    vector.setRowId(0);
    PartitionKey key1 = new PartitionKey(0, 0, 0, 0, 1, 5);
    PartitionKey key2 = new PartitionKey(1, 0, 0, 5, 1, 10);
    List<PartitionKey> keys = new ArrayList<>();
    keys.add(key1);
    keys.add(key2);
    HashMap<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, keys);
    Assert.assertEquals(keys.size(), splits.size());
    int[] offset1 = { 0, 2, 4 };
    int[] values1 = { 0, 2, 4 };
    SparseIntRowUpdateSplit split1 = (SparseIntRowUpdateSplit) splits.get(key1);
    Assert.assertNotNull(split1);
    Assert.assertEquals(offset1.length, split1.size());
    Assert.assertEquals(0, split1.getStart());
    Assert.assertEquals(3, split1.getEnd());
    Assert.assertArrayEquals(offset1, Arrays.copyOfRange(split1.getOffsets(), (int) split1.getStart(), (int) split1.getEnd()));
    for (int i = 0; i < split1.size(); i++) {
        Assert.assertEquals(values1[i], split1.getValues()[(int) split1.getStart() + i], 0.0);
    }
    int[] offset2 = { 6, 8 };
    int[] values2 = { 6, 8 };
    SparseIntRowUpdateSplit split2 = (SparseIntRowUpdateSplit) splits.get(key2);
    Assert.assertNotNull(split2);
    Assert.assertEquals(offset2.length, split2.size());
    Assert.assertEquals(3, split2.getStart());
    Assert.assertEquals(5, split2.getEnd());
    Assert.assertArrayEquals(offset2, Arrays.copyOfRange(split2.getOffsets(), (int) split2.getStart(), (int) split2.getEnd()));
    for (int i = 0; i < split2.size(); i++) {
        Assert.assertEquals(values2[i], split2.getValues()[(int) split2.getStart() + i], 0.0);
    }
    LOG.info("Pass sparseIntVector split Test");
}

Example 39 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class VectorSplitTest method denseVectorSplit.

@Test
public void denseVectorSplit() {
    double[] values = { 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 };
    DenseDoubleVector vector = new DenseDoubleVector(10, values);
    vector.setRowId(0);
    PartitionKey key1 = new PartitionKey(0, 0, 0, 0, 1, 5);
    PartitionKey key2 = new PartitionKey(1, 0, 0, 5, 1, 10);
    List<PartitionKey> keys = new ArrayList<>();
    keys.add(key1);
    keys.add(key2);
    HashMap<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, keys);
    Assert.assertEquals(keys.size(), splits.size());
    double[] values1 = { 0.0, 1.0, 2.0, 3.0, 4.0 };
    DenseDoubleRowUpdateSplit split1 = (DenseDoubleRowUpdateSplit) splits.get(key1);
    Assert.assertNotNull(split1);
    Assert.assertEquals(values1.length, split1.size());
    Assert.assertEquals(0, split1.getStart());
    Assert.assertEquals(5, split1.getEnd());
    for (int i = 0; i < split1.size(); i++) {
        Assert.assertEquals(values1[i], split1.getValues()[(int) split1.getStart() + i], 0.0);
    }
    double[] values2 = { 5.0, 6.0, 7.0, 8.0, 9.0 };
    DenseDoubleRowUpdateSplit split2 = (DenseDoubleRowUpdateSplit) splits.get(key2);
    Assert.assertNotNull(split2);
    Assert.assertEquals(values2.length, split2.size());
    Assert.assertEquals(5, split2.getStart());
    Assert.assertEquals(10, split2.getEnd());
    for (int i = 0; i < split2.size(); i++) {
        Assert.assertEquals(values2[i], split2.getValues()[(int) split2.getStart() + i], 0.0);
    }
    LOG.info("Pass denseVectorSplit Test");
}

Example 40 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class VectorSplitTest method denseIntVectorSplit.

@Test
public void denseIntVectorSplit() {
    int[] values = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    DenseIntVector vector = new DenseIntVector(10, values);
    vector.setRowId(0);
    PartitionKey key1 = new PartitionKey(0, 0, 0, 0, 1, 5);
    PartitionKey key2 = new PartitionKey(1, 0, 0, 5, 1, 10);
    List<PartitionKey> keys = new ArrayList<>();
    keys.add(key1);
    keys.add(key2);
    HashMap<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, keys);
    Assert.assertEquals(keys.size(), splits.size());
    int[] values1 = { 0, 1, 2, 3, 4 };
    DenseIntRowUpdateSplit split1 = (DenseIntRowUpdateSplit) splits.get(key1);
    Assert.assertNotNull(split1);
    Assert.assertEquals(values1.length, split1.size());
    Assert.assertEquals(0, split1.getStart());
    Assert.assertEquals(5, split1.getEnd());
    for (int i = 0; i < split1.size(); i++) {
        Assert.assertEquals(values1[i], split1.getValues()[(int) split1.getStart() + i]);
    }
    int[] values2 = { 5, 6, 7, 8, 9 };
    DenseIntRowUpdateSplit split2 = (DenseIntRowUpdateSplit) splits.get(key2);
    Assert.assertNotNull(split2);
    Assert.assertEquals(values2.length, split2.size());
    Assert.assertEquals(5, split2.getStart());
    Assert.assertEquals(10, split2.getEnd());
    for (int i = 0; i < split2.size(); i++) {
        Assert.assertEquals(values2[i], split2.getValues()[(int) split2.getStart() + i]);
    }
    LOG.info("Pass denseIntVectorSplit Test");
}