use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class PSFailedReportTest method testPSFailedReport.
@Test
public void testPSFailedReport() throws Exception {
ParameterServerId ps1Id = new ParameterServerId(0);
final ParameterServerId ps2Id = new ParameterServerId(1);
PSAttemptId ps1Attempt0Id = new PSAttemptId(ps1Id, 0);
PSAttemptId ps2Attempt0Id = new PSAttemptId(ps2Id, 0);
PSAttemptId ps2Attempt1Id = new PSAttemptId(ps2Id, 1);
ParameterServer ps1Attempt0 = LocalClusterContext.get().getPS(ps1Attempt0Id).getPS();
ParameterServer ps2Attempt0 = LocalClusterContext.get().getPS(ps2Attempt0Id).getPS();
WorkerId worker0Id = new WorkerId(new WorkerGroupId(0), 0);
WorkerAttemptId worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
Worker worker0 = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
TaskContext task0Context = worker0.getTaskManager().getRunningTask().get(task0Id).getTaskContext();
MatrixClient matrixClient = task0Context.getMatrix("w1");
int iterNum = 20;
for (int i = 0; i < iterNum; i++) {
DenseIntVector update = new DenseIntVector(dim);
for (int j = 0; j < dim; j++) {
update.set(j, 1);
}
update.setMatrixId(matrixClient.getMatrixId());
update.setRowId(0);
matrixClient.increment(update);
matrixClient.clock().get();
Thread.sleep(1000);
MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps1w1.getPartition(0));
assertNotNull(ps1w1.getPartition(1));
IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
int part0Size = ps1w1.getRow(0, 0).size();
IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
int part1Size = ps1w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
MatrixStorageManager ps2Storage = ps2Attempt0.getMatrixStorageManager();
ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps2w1.getPartition(0));
assertNotNull(ps2w1.getPartition(1));
row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
part0Size = ps2w1.getRow(0, 0).size();
row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
part1Size = ps2w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
}
LOG.info("===================================================================ps2 failed");
HashMap<PSLocation, Integer> failedCounters = new HashMap<>();
PSLocation psLoc = new PSLocation(ps2Id, ps2Attempt0.getLocationManager().getPsLocation(ps2Id));
failedCounters.put(psLoc, 10000);
worker0.getPSAgent().getMasterClient().psFailedReport(failedCounters);
Thread.sleep(20000);
for (int i = iterNum; i < 2 * iterNum; i++) {
DenseIntVector update = new DenseIntVector(dim);
for (int j = 0; j < dim; j++) {
update.set(j, 1);
}
update.setMatrixId(matrixClient.getMatrixId());
update.setRowId(0);
matrixClient.increment(update);
matrixClient.clock().get();
Thread.sleep(1000);
MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps1w1.getPartition(0));
assertNotNull(ps1w1.getPartition(1));
IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
int part0Size = ps1w1.getRow(0, 0).size();
IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
int part1Size = ps1w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
}
ParameterServer ps2Attempt = LocalClusterContext.get().getPS(ps2Attempt1Id).getPS();
for (int i = iterNum * 2; i < 3 * iterNum; i++) {
DenseIntVector update = new DenseIntVector(dim);
for (int j = 0; j < dim; j++) {
update.set(j, 1);
}
update.setMatrixId(matrixClient.getMatrixId());
update.setRowId(0);
matrixClient.increment(update);
matrixClient.clock().get();
Thread.sleep(1000);
MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps1w1.getPartition(0));
assertNotNull(ps1w1.getPartition(1));
IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
int part0Size = ps1w1.getRow(0, 0).size();
IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
int part1Size = ps1w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
MatrixStorageManager ps2Storage = ps2Attempt.getMatrixStorageManager();
ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps2w1.getPartition(0));
assertNotNull(ps2w1.getPartition(1));
row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
part0Size = ps2w1.getRow(0, 0).size();
row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
part1Size = ps2w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
}
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class IndexGetRowHashTest method testSparseFloatUDF.
public void testSparseFloatUDF() throws Exception {
Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_FLOAT_MAT, 0);
int matrixW1Id = client1.getMatrixId();
int[] index = genIndexs(feaNum, nnz);
IntFloatVector deltaVec = new IntFloatVector(feaNum, new IntFloatSparseVectorStorage(feaNum, nnz));
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
// for (int i = 0; i < feaNum; i++) {
// deltaVec.set(i, i);
// }
deltaVec.setRowId(0);
client1.increment(deltaVec);
client1.clock().get();
// IndexGet func = new IndexGet(new IndexGetParam(matrixW1Id, 0, index));
// IntFloatVector row = (IntFloatVector) ((GetRowResult) client1.get(func)).getRow();
IntFloatVector row = (IntFloatVector) client1.get(0, index);
for (int id : index) {
// System.out.println("id=" + id + ", value=" + row.get(id));
assertEquals(row.get(id), deltaVec.get(id), 0.000001);
}
Assert.assertTrue(index.length == row.size());
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class IndexGetRowsHashTest method testSparseFloatLongKeyUDF.
public void testSparseFloatLongKeyUDF() throws Exception {
Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_FLOAT_LONG_MAT, 0);
int matrixW1Id = client1.getMatrixId();
long[] index = genLongIndexs(feaNum, nnz);
LongFloatVector deltaVec = null;
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new LongFloatVector(feaNum, new LongFloatSparseVectorStorage(feaNum));
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
client1.increment(rowId, deltaVec, true);
}
int[] rowIds = new int[rowNum];
for (int i = 0; i < rowNum; i++) {
rowIds[i] = i;
}
Vector[] rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (long id : index) {
Assert.assertEquals(((LongFloatVector) rows[i]).get(id), deltaVec.get(id), zero);
}
}
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class IndexGetRowsHashTest method testSparseDoubleUDF.
public void testSparseDoubleUDF() throws Exception {
Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_DOUBLE_MAT, 0);
int[] index = genIndexs(feaNum, nnz);
IntDoubleVector deltaVec = null;
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new IntDoubleVector(feaNum, new IntDoubleSparseVectorStorage(feaNum));
for (int i = 0; i < index.length; i++) {
deltaVec.set(index[i], index[i]);
}
client1.increment(rowId, deltaVec, true);
}
int[] rowIds = new int[rowNum];
for (int i = 0; i < rowNum; i++) {
rowIds[i] = i;
}
Vector[] rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntDoubleVector) rows[i]).get(id), deltaVec.get(id), zero);
}
}
}
use of com.tencent.angel.worker.Worker in project angel by Tencent.
the class IndexGetRowsHashTest method testDenseFloatUDF.
public void testDenseFloatUDF() throws Exception {
Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
MatrixClient client1 = worker.getPSAgent().getMatrixClient(DENSE_FLOAT_MAT, 0);
int[] index = genIndexs(feaNum, nnz);
IntFloatVector deltaVec = null;
for (int rowId = 0; rowId < rowNum; rowId++) {
deltaVec = new IntFloatVector(feaNum, new IntFloatDenseVectorStorage(feaNum));
for (int i = 0; i < feaNum; i++) deltaVec.set(i, i);
client1.increment(rowId, deltaVec, true);
}
int[] rowIds = new int[rowNum];
for (int i = 0; i < rowNum; i++) {
rowIds[i] = i;
}
Vector[] rows = client1.get(rowIds, index);
for (int i = 0; i < rowNum; i++) {
for (int id : index) {
Assert.assertEquals(((IntFloatVector) rows[i]).get(id), deltaVec.get(id), zero);
}
}
}
Aggregations