use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class UpdateRowsTest method testSparseDoubleLongKeyUDF.
public void testSparseDoubleLongKeyUDF() throws Exception {
Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_DOUBLE_LONG_MAT, 0);
int matrixW1Id = client1.getMatrixId();
long[] index = genLongIndexs(feaNum, nnz);
LongDoubleVector deltaVec = null;
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new LongDoubleVector(feaNum, new LongDoubleSparseVectorStorage(feaNum));
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
client1.update(rowId, deltaVec);
}
int[] rowIds = new int[rowNum];
for (int i = 0; i < rowNum; i++) {
rowIds[i] = i;
}
Vector[] rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (long id : index) {
Assert.assertEquals(((LongDoubleVector) rows[i]).get(id), deltaVec.get(id), zero);
}
}
client1.zero();
LongDoubleVector[] deltaVecs = new LongDoubleVector[rowNum];
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new LongDoubleVector(feaNum, new LongDoubleSparseVectorStorage(feaNum));
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
deltaVecs[rowId] = deltaVec;
}
client1.update(rowIds, deltaVecs);
rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (long id : index) {
Assert.assertEquals(((LongDoubleVector) rows[i]).get(id), deltaVecs[i].get(id), zero);
}
}
client1.zero();
deltaVecs = new LongDoubleVector[rowNum];
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new LongDoubleVector(feaNum, new LongDoubleSparseVectorStorage(feaNum));
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
deltaVecs[rowId] = deltaVec;
}
client1.increment(rowIds, deltaVecs, true);
rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (long id : index) {
Assert.assertEquals(((LongDoubleVector) rows[i]).get(id), deltaVecs[i].get(id), zero);
}
}
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class UpdateRowsTest method testSparseIntUDF.
public void testSparseIntUDF() throws Exception {
Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_INT_MAT, 0);
int matrixW1Id = client1.getMatrixId();
int[] index = genIndexs(feaNum, nnz);
IntIntVector deltaVec = null;
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new IntIntVector(feaNum, new IntIntSparseVectorStorage(feaNum));
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
client1.update(rowId, deltaVec);
}
int[] rowIds = new int[rowNum];
for (int i = 0; i < rowNum; i++) {
rowIds[i] = i;
}
Vector[] rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntIntVector) rows[i]).get(id), deltaVec.get(id), zero);
}
Assert.assertTrue(index.length == ((IntIntVector) rows[i]).size());
}
client1.zero();
IntIntVector[] deltaVecs = new IntIntVector[rowNum];
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = VFactory.sparseIntVector(feaNum, index.length);
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
deltaVecs[rowId] = deltaVec;
}
client1.update(rowIds, deltaVecs);
rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntIntVector) rows[i]).get(id), deltaVecs[i].get(id), zero);
}
}
client1.zero();
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = VFactory.sparseIntVector(feaNum, index.length);
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
deltaVecs[rowId] = deltaVec;
}
client1.increment(rowIds, deltaVecs, true);
rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntIntVector) rows[i]).get(id), deltaVecs[i].get(id), zero);
}
}
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class UpdateRowsTest method testDenseFloatUDF.
public void testDenseFloatUDF() throws Exception {
Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
MatrixClient client1 = worker.getPSAgent().getMatrixClient(DENSE_FLOAT_MAT, 0);
int[] index = genIndexs(feaNum, nnz);
IntFloatVector deltaVec = null;
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new IntFloatVector(feaNum, new IntFloatDenseVectorStorage(feaNum));
for (int i = 0; i < feaNum; i++) deltaVec.set(i, i);
client1.update(rowId, deltaVec);
}
int[] rowIds = new int[rowNum];
for (int i = 0; i < rowNum; i++) {
rowIds[i] = i;
}
Vector[] rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntFloatVector) rows[i]).get(id), deltaVec.get(id), zero);
}
}
client1.zero();
IntFloatVector[] deltaVecs = new IntFloatVector[rowNum];
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = VFactory.sparseFloatVector(feaNum, index.length);
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
deltaVecs[rowId] = deltaVec;
}
client1.update(rowIds, deltaVecs);
rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntFloatVector) rows[i]).get(id), deltaVecs[i].get(id), zero);
}
}
client1.zero();
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = VFactory.sparseFloatVector(feaNum, index.length);
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
deltaVecs[rowId] = deltaVec;
}
client1.increment(rowIds, deltaVecs, true);
rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntFloatVector) rows[i]).get(id), deltaVecs[i].get(id), zero);
}
}
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class MatrixMetaManagerTest method testCreateMatrix.
@Test
public void testCreateMatrix() throws Exception {
try {
LOG.info("===========================testCreateMatrix===============================");
Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
MasterClient masterClient = worker.getPSAgent().getMasterClient();
int w3Id = -1;
int w4Id = -1;
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w3");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, RowType.T_DOUBLE_DENSE.name());
masterClient.createMatrix(mMatrix, 10000);
mMatrix.setName("w4");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, RowType.T_DOUBLE_DENSE.name());
masterClient.createMatrix(mMatrix, 10000);
MatrixMeta w3Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w3");
MatrixMeta w4Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w4");
assertEquals(w3Meta.getRowNum(), 1);
assertEquals(w3Meta.getColNum(), 100000);
assertEquals(w3Meta.getRowType(), RowType.T_DOUBLE_DENSE);
assertEquals(w4Meta.getRowNum(), 1);
assertEquals(w4Meta.getColNum(), 100000);
assertEquals(w4Meta.getRowType(), RowType.T_DOUBLE_DENSE);
w3Id = w3Meta.getId();
w4Id = w4Meta.getId();
AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
assertTrue(angelAppMaster != null);
AMMatrixMetaManager matrixMetaManager = angelAppMaster.getAppContext().getMatrixMetaManager();
MatrixMeta matrixw3Proto = matrixMetaManager.getMatrix("w3");
MatrixMeta matrixw4Proto = matrixMetaManager.getMatrix("w4");
assertNotNull(matrixw3Proto);
assertNotNull(matrixw4Proto);
assertEquals(matrixw3Proto.getRowNum(), 1);
assertEquals(matrixw3Proto.getColNum(), 100000);
assertEquals(matrixw3Proto.getPartitionMetas().size(), 2);
Map<Integer, PartitionMeta> w3Parts = matrixw3Proto.getPartitionMetas();
assertEquals(w3Parts.get(0).getPss().get(0), psId);
assertEquals(w3Parts.get(0).getPartId(), 0);
assertEquals(w3Parts.get(0).getStartRow(), 0);
assertEquals(w3Parts.get(0).getEndRow(), 1);
assertEquals(w3Parts.get(0).getStartCol(), 0);
assertEquals(w3Parts.get(0).getEndCol(), 50000);
assertEquals(w3Parts.get(1).getPartId(), 1);
assertEquals(w3Parts.get(1).getStartRow(), 0);
assertEquals(w3Parts.get(1).getEndRow(), 1);
assertEquals(w3Parts.get(1).getStartCol(), 50000);
assertEquals(w3Parts.get(1).getEndCol(), 100000);
Map<Integer, PartitionMeta> w4Parts = matrixw4Proto.getPartitionMetas();
assertEquals(w4Parts.get(0).getPss().get(0), psId);
assertEquals(w4Parts.get(0).getPartId(), 0);
assertEquals(w4Parts.get(0).getStartRow(), 0);
assertEquals(w4Parts.get(0).getEndRow(), 1);
assertEquals(w4Parts.get(0).getStartCol(), 0);
assertEquals(w4Parts.get(0).getEndCol(), 50000);
assertEquals(w4Parts.get(1).getPartId(), 1);
assertEquals(w4Parts.get(1).getStartRow(), 0);
assertEquals(w4Parts.get(1).getEndRow(), 1);
assertEquals(w4Parts.get(1).getStartCol(), 50000);
assertEquals(w4Parts.get(1).getEndCol(), 100000);
ParameterServer ps = LocalClusterContext.get().getPS(psAttempt0Id).getPS();
PSMatrixMetaManager matrixPartManager = ps.getMatrixMetaManager();
PartitionMeta w3Part0 = matrixPartManager.getPartMeta(w3Id, 0);
PartitionMeta w3Part1 = matrixPartManager.getPartMeta(w3Id, 1);
assertTrue(w3Part0 != null);
assertTrue(w3Part1 != null);
assertEquals(w3Part0.getPartitionKey().getStartRow(), 0);
assertEquals(w3Part0.getPartitionKey().getEndRow(), 1);
assertEquals(w3Part0.getPartitionKey().getStartCol(), 0);
assertEquals(w3Part0.getPartitionKey().getEndCol(), 50000);
assertEquals(w3Part1.getPartitionKey().getStartRow(), 0);
assertEquals(w3Part1.getPartitionKey().getEndRow(), 1);
assertEquals(w3Part1.getPartitionKey().getStartCol(), 50000);
assertEquals(w3Part1.getPartitionKey().getEndCol(), 100000);
PartitionMeta w4Part0 = matrixPartManager.getPartMeta(w4Id, 0);
PartitionMeta w4Part1 = matrixPartManager.getPartMeta(w4Id, 1);
assertTrue(w4Part0 != null);
assertTrue(w4Part1 != null);
assertEquals(w4Part0.getPartitionKey().getStartRow(), 0);
assertEquals(w4Part0.getPartitionKey().getEndRow(), 1);
assertEquals(w4Part0.getPartitionKey().getStartCol(), 0);
assertEquals(w4Part0.getPartitionKey().getEndCol(), 50000);
assertEquals(w4Part1.getPartitionKey().getStartRow(), 0);
assertEquals(w4Part1.getPartitionKey().getEndRow(), 1);
assertEquals(w4Part1.getPartitionKey().getStartCol(), 50000);
assertEquals(w4Part1.getPartitionKey().getEndCol(), 100000);
MatrixClient w4ClientForTask0 = worker.getPSAgent().getMatrixClient("w4", 0);
MatrixClient w4ClientForTask1 = worker.getPSAgent().getMatrixClient("w4", 1);
TaskContext task0Context = w4ClientForTask0.getTaskContext();
TaskContext task1Context = w4ClientForTask1.getTaskContext();
double[] delta = new double[100000];
for (int i = 0; i < delta.length; i++) {
delta[i] = 1.0;
}
int iterIndex = 0;
while (iterIndex < 5) {
IntDoubleVector row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
double sum1 = sum(row1.getStorage().getValues());
LOG.info("taskid=" + task0Context.getIndex() + ", matrixId=" + w4ClientForTask0.getMatrixId() + ", rowIndex=0, local row sum=" + sum1);
IntDoubleVector deltaRow1 = new IntDoubleVector(delta.length, new IntDoubleDenseVectorStorage(delta));
deltaRow1.setMatrixId(w4ClientForTask0.getMatrixId());
deltaRow1.setRowId(0);
w4ClientForTask0.increment(deltaRow1);
w4ClientForTask0.clock().get();
task0Context.increaseEpoch();
IntDoubleVector row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
double sum2 = sum(row2.getStorage().getValues());
LOG.info("taskid=" + task1Context.getIndex() + ", matrixId=" + w4ClientForTask1.getMatrixId() + ", rowIndex=1, local row sum=" + sum2);
IntDoubleVector deltaRow2 = new IntDoubleVector(delta.length, new IntDoubleDenseVectorStorage(delta));
deltaRow2.setMatrixId(w4ClientForTask1.getMatrixId());
deltaRow2.setRowId(0);
w4ClientForTask1.increment(deltaRow2);
w4ClientForTask1.clock().get();
task1Context.increaseEpoch();
iterIndex++;
}
AMTaskManager amTaskManager = angelAppMaster.getAppContext().getTaskManager();
AMTask amTask0 = amTaskManager.getTask(task0Id);
AMTask amTask1 = amTaskManager.getTask(task1Id);
assertEquals(amTask0.getIteration(), 5);
assertEquals(amTask1.getIteration(), 5);
Int2IntOpenHashMap task0MatrixClocks = amTask0.getMatrixClocks();
assertEquals(task0MatrixClocks.size(), 1);
assertEquals(task0MatrixClocks.get(w4Id), 5);
Int2IntOpenHashMap task1MatrixClocks = amTask1.getMatrixClocks();
assertEquals(task1MatrixClocks.size(), 1);
assertEquals(task1MatrixClocks.get(w4Id), 5);
IntDoubleVector row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
double sum1 = sum(row1.getStorage().getValues());
assertEquals(sum1, 1000000.0, 0.000001);
IntDoubleVector row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
double sum2 = sum(row2.getStorage().getValues());
assertEquals(sum2, 1000000.0, 0.000001);
masterClient.releaseMatrix(w3Meta.getName());
Thread.sleep(10000);
matrixw3Proto = matrixMetaManager.getMatrix("w3");
assertTrue(matrixw3Proto == null);
MatrixStorageManager matrixStorageManager = LocalClusterContext.get().getPS(psAttempt0Id).getPS().getMatrixStorageManager();
ServerMatrix sw3 = matrixStorageManager.getMatrix(w3Id);
assertTrue(sw3 == null);
w4ClientForTask0.clock().get();
w4ClientForTask1.clock().get();
row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
sum1 = sum(row1.getStorage().getValues());
assertEquals(sum1, 1000000.0, 0.000001);
row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
sum2 = sum(row2.getStorage().getValues());
assertEquals(sum2, 1000000.0, 0.000001);
} catch (Exception x) {
LOG.error("run testCreateMatrix failed ", x);
throw x;
}
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class TaskManagerTest method testTaskMatrixClock.
@Test
public void testTaskMatrixClock() throws ServiceException {
try {
LOG.info("===========================testTaskMatrixClock===============================");
AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
assertTrue(angelAppMaster != null);
AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
PSAgentMatrixMetaManager matrixMetaManager = worker.getPSAgent().getMatrixMetaManager();
int w1Id = matrixMetaManager.getMatrixId("w1");
int w2Id = matrixMetaManager.getMatrixId("w2");
MasterClient masterClient = worker.getPSAgent().getMasterClient();
AMTask task0 = taskManager.getTask(task0Id);
AMTask task1 = taskManager.getTask(task1Id);
masterClient.updateClock(task0Id.getIndex(), w1Id, 1);
masterClient.updateClock(task0Id.getIndex(), w2Id, 1);
Int2IntOpenHashMap matrixClocks = task0.getMatrixClocks();
assertEquals(matrixClocks.size(), 2);
assertEquals(matrixClocks.get(w1Id), 1);
assertEquals(matrixClocks.get(w2Id), 1);
masterClient.updateClock(task0Id.getIndex(), w1Id, 2);
assertEquals(task0.getMatrixClock(w1Id), 2);
assertEquals(task0.getMatrixClock(w2Id), 1);
masterClient.updateClock(task1Id.getIndex(), w1Id, 1);
masterClient.updateClock(task1Id.getIndex(), w2Id, 1);
matrixClocks = task1.getMatrixClocks();
assertEquals(matrixClocks.size(), 2);
assertEquals(matrixClocks.get(w1Id), 1);
assertEquals(matrixClocks.get(w2Id), 1);
masterClient.updateClock(task1Id.getIndex(), w1Id, 2);
assertEquals(task1.getMatrixClock(w1Id), 2);
assertEquals(task1.getMatrixClock(w2Id), 1);
} catch (Exception x) {
LOG.error("run testTaskMatrixClock failed ", x);
throw x;
}
}
Aggregations