use of com.tencent.angel.PartitionKey in project angel by Tencent.
the class ServerPartitionTest method setUp.
@Before
public void setUp() throws Exception {
partitionId = 1;
matrixId = 1;
startRow = 2;
startCol = 2;
endRow = 8;
endCol = 10;
rowType = RowType.T_DOUBLE_DENSE;
partitionKey = new PartitionKey(partitionId, matrixId, startRow, startCol, endRow, endCol);
serverPartition = new ServerPartition(partitionKey, rowType);
serverPartition.init();
}
use of com.tencent.angel.PartitionKey in project angel by Tencent.
the class PSAgentTest method testMatrixLocationManager.
@Test
public void testMatrixLocationManager() throws Exception {
try {
AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
assertTrue(angelAppMaster != null);
AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
assertTrue(taskManager != null);
WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
assertTrue(workerManager != null);
Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
assertTrue(worker != null);
PSAgent psAgent = worker.getPSAgent();
assertTrue(psAgent != null);
PSAgentMatrixMetaManager matrixPartitionRouter = psAgent.getMatrixMetaManager();
PSAgentLocationManager locationCache = psAgent.getLocationManager();
assertTrue(matrixPartitionRouter != null);
// test ps location
Location psLoc = locationCache.getPsLocation(psId);
String ipRegex = "(2[5][0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})";
Pattern pattern = Pattern.compile(ipRegex);
Matcher matcher = pattern.matcher(psLoc.getIp());
assertTrue(matcher.matches());
assertTrue(psLoc.getPort() >= 1 && psLoc.getPort() <= 65535);
int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
// test partitions
List<PartitionKey> partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id);
assertEquals(partition1Keys.size(), 2);
List<PartitionKey> partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id);
assertEquals(partition2Keys.size(), 2);
partition1Keys.clear();
partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
assertEquals(partition1Keys.size(), 2);
partition2Keys.clear();
partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
assertEquals(partition2Keys.size(), 2);
int rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
assertEquals(rowPartSize, 2);
rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
assertEquals(rowPartSize, 2);
} catch (Exception x) {
LOG.error("run testMatrixLocationManager failed ", x);
throw x;
}
}
use of com.tencent.angel.PartitionKey in project angel by Tencent.
the class VectorSplitTest method sparseIntVectorSplit.
@Test
public void sparseIntVectorSplit() {
int[] offsets = { 0, 2, 4, 6, 8 };
int[] values = { 0, 2, 4, 6, 8 };
SparseIntVector vector = new SparseIntVector(10, offsets, values);
vector.setRowId(0);
PartitionKey key1 = new PartitionKey(0, 0, 0, 0, 1, 5);
PartitionKey key2 = new PartitionKey(1, 0, 0, 5, 1, 10);
List<PartitionKey> keys = new ArrayList<>();
keys.add(key1);
keys.add(key2);
HashMap<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, keys);
Assert.assertEquals(keys.size(), splits.size());
int[] offset1 = { 0, 2, 4 };
int[] values1 = { 0, 2, 4 };
SparseIntRowUpdateSplit split1 = (SparseIntRowUpdateSplit) splits.get(key1);
Assert.assertNotNull(split1);
Assert.assertEquals(offset1.length, split1.size());
Assert.assertEquals(0, split1.getStart());
Assert.assertEquals(3, split1.getEnd());
Assert.assertArrayEquals(offset1, Arrays.copyOfRange(split1.getOffsets(), (int) split1.getStart(), (int) split1.getEnd()));
for (int i = 0; i < split1.size(); i++) {
Assert.assertEquals(values1[i], split1.getValues()[(int) split1.getStart() + i], 0.0);
}
int[] offset2 = { 6, 8 };
int[] values2 = { 6, 8 };
SparseIntRowUpdateSplit split2 = (SparseIntRowUpdateSplit) splits.get(key2);
Assert.assertNotNull(split2);
Assert.assertEquals(offset2.length, split2.size());
Assert.assertEquals(3, split2.getStart());
Assert.assertEquals(5, split2.getEnd());
Assert.assertArrayEquals(offset2, Arrays.copyOfRange(split2.getOffsets(), (int) split2.getStart(), (int) split2.getEnd()));
for (int i = 0; i < split2.size(); i++) {
Assert.assertEquals(values2[i], split2.getValues()[(int) split2.getStart() + i], 0.0);
}
LOG.info("Pass sparseIntVector split Test");
}
use of com.tencent.angel.PartitionKey in project angel by Tencent.
the class VectorSplitTest method denseVectorSplit.
@Test
public void denseVectorSplit() {
double[] values = { 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 };
DenseDoubleVector vector = new DenseDoubleVector(10, values);
vector.setRowId(0);
PartitionKey key1 = new PartitionKey(0, 0, 0, 0, 1, 5);
PartitionKey key2 = new PartitionKey(1, 0, 0, 5, 1, 10);
List<PartitionKey> keys = new ArrayList<>();
keys.add(key1);
keys.add(key2);
HashMap<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, keys);
Assert.assertEquals(keys.size(), splits.size());
double[] values1 = { 0.0, 1.0, 2.0, 3.0, 4.0 };
DenseDoubleRowUpdateSplit split1 = (DenseDoubleRowUpdateSplit) splits.get(key1);
Assert.assertNotNull(split1);
Assert.assertEquals(values1.length, split1.size());
Assert.assertEquals(0, split1.getStart());
Assert.assertEquals(5, split1.getEnd());
for (int i = 0; i < split1.size(); i++) {
Assert.assertEquals(values1[i], split1.getValues()[(int) split1.getStart() + i], 0.0);
}
double[] values2 = { 5.0, 6.0, 7.0, 8.0, 9.0 };
DenseDoubleRowUpdateSplit split2 = (DenseDoubleRowUpdateSplit) splits.get(key2);
Assert.assertNotNull(split2);
Assert.assertEquals(values2.length, split2.size());
Assert.assertEquals(5, split2.getStart());
Assert.assertEquals(10, split2.getEnd());
for (int i = 0; i < split2.size(); i++) {
Assert.assertEquals(values2[i], split2.getValues()[(int) split2.getStart() + i], 0.0);
}
LOG.info("Pass denseVectorSplit Test");
}
use of com.tencent.angel.PartitionKey in project angel by Tencent.
the class VectorSplitTest method denseIntVectorSplit.
@Test
public void denseIntVectorSplit() {
int[] values = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
DenseIntVector vector = new DenseIntVector(10, values);
vector.setRowId(0);
PartitionKey key1 = new PartitionKey(0, 0, 0, 0, 1, 5);
PartitionKey key2 = new PartitionKey(1, 0, 0, 5, 1, 10);
List<PartitionKey> keys = new ArrayList<>();
keys.add(key1);
keys.add(key2);
HashMap<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, keys);
Assert.assertEquals(keys.size(), splits.size());
int[] values1 = { 0, 1, 2, 3, 4 };
DenseIntRowUpdateSplit split1 = (DenseIntRowUpdateSplit) splits.get(key1);
Assert.assertNotNull(split1);
Assert.assertEquals(values1.length, split1.size());
Assert.assertEquals(0, split1.getStart());
Assert.assertEquals(5, split1.getEnd());
for (int i = 0; i < split1.size(); i++) {
Assert.assertEquals(values1[i], split1.getValues()[(int) split1.getStart() + i]);
}
int[] values2 = { 5, 6, 7, 8, 9 };
DenseIntRowUpdateSplit split2 = (DenseIntRowUpdateSplit) splits.get(key2);
Assert.assertNotNull(split2);
Assert.assertEquals(values2.length, split2.size());
Assert.assertEquals(5, split2.getStart());
Assert.assertEquals(10, split2.getEnd());
for (int i = 0; i < split2.size(); i++) {
Assert.assertEquals(values2[i], split2.getValues()[(int) split2.getStart() + i]);
}
LOG.info("Pass denseIntVectorSplit Test");
}
Aggregations