use of com.tencent.angel.ps.impl.matrix.ServerMatrix in project angel by Tencent.
the class MatrixStorageManager method loadMatrix.
private void loadMatrix(Path inputPath, int matrixId) throws IOException {
ServerMatrix matrix = context.getMatrixStorageManager().getMatrix(matrixId);
MatrixMeta matrixMeta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
if (matrix != null) {
if (inputPath == null) {
try {
inputPath = recover.getSnapshotPath(matrixId);
} catch (IOException e) {
LOG.error("Get snapshot path failed, ", e);
}
if (inputPath == null) {
String loadPathStr = matrixMeta.getAttribute(MatrixConf.MATRIX_LOAD_PATH);
if (loadPathStr != null) {
inputPath = new Path(loadPathStr);
}
}
}
matrix.load(matrixMeta, inputPath);
}
}
use of com.tencent.angel.ps.impl.matrix.ServerMatrix in project angel by Tencent.
the class MatrixStorageManager method getMatrixNames.
private String getMatrixNames(Collection<Integer> matrixIds) {
int size = matrixIds.size();
StringBuilder sb = new StringBuilder();
int i = 0;
for (int matrixId : matrixIds) {
ServerMatrix matrix = getMatrix(matrixId);
if (matrix != null) {
sb.append(matrix.getName());
} else {
sb.append("null");
}
if (i < size - 1) {
sb.append(",");
}
i++;
}
return sb.toString();
}
use of com.tencent.angel.ps.impl.matrix.ServerMatrix in project angel by Tencent.
the class MatrixMetaManagerTest method testCreateMatrix.
@Test
public void testCreateMatrix() throws Exception {
try {
LOG.info("===========================testCreateMatrix===============================");
Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
MasterClient masterClient = worker.getPSAgent().getMasterClient();
int w3Id = -1;
int w4Id = -1;
// add matrix
MatrixContext mMatrix = new MatrixContext();
mMatrix.setName("w3");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
masterClient.createMatrix(mMatrix, 10000);
mMatrix.setName("w4");
mMatrix.setRowNum(1);
mMatrix.setColNum(100000);
mMatrix.setMaxRowNumInBlock(1);
mMatrix.setMaxColNumInBlock(50000);
mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
masterClient.createMatrix(mMatrix, 10000);
MatrixMeta w3Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w3");
MatrixMeta w4Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w4");
assertEquals(w3Meta.getRowNum(), 1);
assertEquals(w3Meta.getColNum(), 100000);
assertEquals(w3Meta.getRowType(), RowType.T_DOUBLE_DENSE);
assertEquals(w4Meta.getRowNum(), 1);
assertEquals(w4Meta.getColNum(), 100000);
assertEquals(w4Meta.getRowType(), RowType.T_DOUBLE_DENSE);
w3Id = w3Meta.getId();
w4Id = w4Meta.getId();
AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
assertTrue(angelAppMaster != null);
AMMatrixMetaManager matrixMetaManager = angelAppMaster.getAppContext().getMatrixMetaManager();
MatrixMeta matrixw3Proto = matrixMetaManager.getMatrix("w3");
MatrixMeta matrixw4Proto = matrixMetaManager.getMatrix("w4");
assertNotNull(matrixw3Proto);
assertNotNull(matrixw4Proto);
assertEquals(matrixw3Proto.getRowNum(), 1);
assertEquals(matrixw3Proto.getColNum(), 100000);
assertEquals(matrixw3Proto.getPartitionMetas().size(), 2);
Map<Integer, PartitionMeta> w3Parts = matrixw3Proto.getPartitionMetas();
assertEquals(w3Parts.get(0).getPss().get(0), psId);
assertEquals(w3Parts.get(0).getPartId(), 0);
assertEquals(w3Parts.get(0).getStartRow(), 0);
assertEquals(w3Parts.get(0).getEndRow(), 1);
assertEquals(w3Parts.get(0).getStartCol(), 0);
assertEquals(w3Parts.get(0).getEndCol(), 50000);
assertEquals(w3Parts.get(1).getPartId(), 1);
assertEquals(w3Parts.get(1).getStartRow(), 0);
assertEquals(w3Parts.get(1).getEndRow(), 1);
assertEquals(w3Parts.get(1).getStartCol(), 50000);
assertEquals(w3Parts.get(1).getEndCol(), 100000);
Map<Integer, PartitionMeta> w4Parts = matrixw4Proto.getPartitionMetas();
assertEquals(w4Parts.get(0).getPss().get(0), psId);
assertEquals(w4Parts.get(0).getPartId(), 0);
assertEquals(w4Parts.get(0).getStartRow(), 0);
assertEquals(w4Parts.get(0).getEndRow(), 1);
assertEquals(w4Parts.get(0).getStartCol(), 0);
assertEquals(w4Parts.get(0).getEndCol(), 50000);
assertEquals(w4Parts.get(1).getPartId(), 1);
assertEquals(w4Parts.get(1).getStartRow(), 0);
assertEquals(w4Parts.get(1).getEndRow(), 1);
assertEquals(w4Parts.get(1).getStartCol(), 50000);
assertEquals(w4Parts.get(1).getEndCol(), 100000);
ParameterServer ps = LocalClusterContext.get().getPS(psAttempt0Id).getPS();
PSMatrixMetaManager matrixPartManager = ps.getMatrixMetaManager();
PartitionMeta w3Part0 = matrixPartManager.getPartMeta(w3Id, 0);
PartitionMeta w3Part1 = matrixPartManager.getPartMeta(w3Id, 1);
assertTrue(w3Part0 != null);
assertTrue(w3Part1 != null);
assertEquals(w3Part0.getPartitionKey().getStartRow(), 0);
assertEquals(w3Part0.getPartitionKey().getEndRow(), 1);
assertEquals(w3Part0.getPartitionKey().getStartCol(), 0);
assertEquals(w3Part0.getPartitionKey().getEndCol(), 50000);
assertEquals(w3Part1.getPartitionKey().getStartRow(), 0);
assertEquals(w3Part1.getPartitionKey().getEndRow(), 1);
assertEquals(w3Part1.getPartitionKey().getStartCol(), 50000);
assertEquals(w3Part1.getPartitionKey().getEndCol(), 100000);
PartitionMeta w4Part0 = matrixPartManager.getPartMeta(w4Id, 0);
PartitionMeta w4Part1 = matrixPartManager.getPartMeta(w4Id, 1);
assertTrue(w4Part0 != null);
assertTrue(w4Part1 != null);
assertEquals(w4Part0.getPartitionKey().getStartRow(), 0);
assertEquals(w4Part0.getPartitionKey().getEndRow(), 1);
assertEquals(w4Part0.getPartitionKey().getStartCol(), 0);
assertEquals(w4Part0.getPartitionKey().getEndCol(), 50000);
assertEquals(w4Part1.getPartitionKey().getStartRow(), 0);
assertEquals(w4Part1.getPartitionKey().getEndRow(), 1);
assertEquals(w4Part1.getPartitionKey().getStartCol(), 50000);
assertEquals(w4Part1.getPartitionKey().getEndCol(), 100000);
MatrixClient w4ClientForTask0 = worker.getPSAgent().getMatrixClient("w4", 0);
MatrixClient w4ClientForTask1 = worker.getPSAgent().getMatrixClient("w4", 1);
TaskContext task0Context = w4ClientForTask0.getTaskContext();
TaskContext task1Context = w4ClientForTask1.getTaskContext();
double[] delta = new double[100000];
for (int i = 0; i < delta.length; i++) {
delta[i] = 1.0;
}
int iterIndex = 0;
while (iterIndex < 5) {
DenseDoubleVector row1 = (DenseDoubleVector) w4ClientForTask0.getRow(0);
double sum1 = sum(row1.getValues());
LOG.info("taskid=" + task0Context.getIndex() + ", matrixId=" + w4ClientForTask0.getMatrixId() + ", rowIndex=0, local row sum=" + sum1);
DenseDoubleVector deltaRow1 = new DenseDoubleVector(delta.length, delta);
deltaRow1.setMatrixId(w4ClientForTask0.getMatrixId());
deltaRow1.setRowId(0);
w4ClientForTask0.increment(deltaRow1);
w4ClientForTask0.clock().get();
task0Context.increaseEpoch();
DenseDoubleVector row2 = (DenseDoubleVector) w4ClientForTask1.getRow(0);
double sum2 = sum(row2.getValues());
LOG.info("taskid=" + task1Context.getIndex() + ", matrixId=" + w4ClientForTask1.getMatrixId() + ", rowIndex=1, local row sum=" + sum2);
DenseDoubleVector deltaRow2 = new DenseDoubleVector(delta.length, delta);
deltaRow2.setMatrixId(w4ClientForTask1.getMatrixId());
deltaRow2.setRowId(0);
w4ClientForTask1.increment(deltaRow2);
w4ClientForTask1.clock().get();
task1Context.increaseEpoch();
iterIndex++;
}
AMTaskManager amTaskManager = angelAppMaster.getAppContext().getTaskManager();
AMTask amTask0 = amTaskManager.getTask(task0Id);
AMTask amTask1 = amTaskManager.getTask(task1Id);
assertEquals(amTask0.getIteration(), 5);
assertEquals(amTask1.getIteration(), 5);
Int2IntOpenHashMap task0MatrixClocks = amTask0.getMatrixClocks();
assertEquals(task0MatrixClocks.size(), 1);
assertEquals(task0MatrixClocks.get(w4Id), 5);
Int2IntOpenHashMap task1MatrixClocks = amTask1.getMatrixClocks();
assertEquals(task1MatrixClocks.size(), 1);
assertEquals(task1MatrixClocks.get(w4Id), 5);
DenseDoubleVector row1 = (DenseDoubleVector) w4ClientForTask0.getRow(0);
double sum1 = sum(row1.getValues());
assertEquals(sum1, 1000000.0, 0.000001);
DenseDoubleVector row2 = (DenseDoubleVector) w4ClientForTask1.getRow(0);
double sum2 = sum(row2.getValues());
assertEquals(sum2, 1000000.0, 0.000001);
masterClient.releaseMatrix(w3Meta.getName());
Thread.sleep(10000);
matrixw3Proto = matrixMetaManager.getMatrix("w3");
assertTrue(matrixw3Proto == null);
MatrixStorageManager matrixStorageManager = LocalClusterContext.get().getPS(psAttempt0Id).getPS().getMatrixStorageManager();
ServerMatrix sw3 = matrixStorageManager.getMatrix(w3Id);
assertTrue(sw3 == null);
w4ClientForTask0.clock().get();
w4ClientForTask1.clock().get();
row1 = (DenseDoubleVector) w4ClientForTask0.getRow(0);
sum1 = sum(row1.getValues());
assertEquals(sum1, 1000000.0, 0.000001);
row2 = (DenseDoubleVector) w4ClientForTask1.getRow(0);
sum2 = sum(row2.getValues());
assertEquals(sum2, 1000000.0, 0.000001);
} catch (Exception x) {
LOG.error("run testCreateMatrix failed ", x);
throw x;
}
}
use of com.tencent.angel.ps.impl.matrix.ServerMatrix in project angel by Tencent.
the class PSFailedReportTest method testPSFailedReport.
@Test
public void testPSFailedReport() throws Exception {
ParameterServerId ps1Id = new ParameterServerId(0);
final ParameterServerId ps2Id = new ParameterServerId(1);
PSAttemptId ps1Attempt0Id = new PSAttemptId(ps1Id, 0);
PSAttemptId ps2Attempt0Id = new PSAttemptId(ps2Id, 0);
PSAttemptId ps2Attempt1Id = new PSAttemptId(ps2Id, 1);
ParameterServer ps1Attempt0 = LocalClusterContext.get().getPS(ps1Attempt0Id).getPS();
ParameterServer ps2Attempt0 = LocalClusterContext.get().getPS(ps2Attempt0Id).getPS();
WorkerId worker0Id = new WorkerId(new WorkerGroupId(0), 0);
WorkerAttemptId worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
Worker worker0 = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
TaskContext task0Context = worker0.getTaskManager().getRunningTask().get(task0Id).getTaskContext();
MatrixClient matrixClient = task0Context.getMatrix("w1");
int iterNum = 20;
for (int i = 0; i < iterNum; i++) {
DenseIntVector update = new DenseIntVector(dim);
for (int j = 0; j < dim; j++) {
update.set(j, 1);
}
update.setMatrixId(matrixClient.getMatrixId());
update.setRowId(0);
matrixClient.increment(update);
matrixClient.clock().get();
Thread.sleep(1000);
MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps1w1.getPartition(0));
assertNotNull(ps1w1.getPartition(1));
IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
int part0Size = ps1w1.getRow(0, 0).size();
IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
int part1Size = ps1w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
MatrixStorageManager ps2Storage = ps2Attempt0.getMatrixStorageManager();
ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps2w1.getPartition(0));
assertNotNull(ps2w1.getPartition(1));
row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
part0Size = ps2w1.getRow(0, 0).size();
row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
part1Size = ps2w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
}
LOG.info("===================================================================ps2 failed");
HashMap<PSLocation, Integer> failedCounters = new HashMap<>();
PSLocation psLoc = new PSLocation(ps2Id, ps2Attempt0.getLocationManager().getPsLocation(ps2Id));
failedCounters.put(psLoc, 10000);
worker0.getPSAgent().getMasterClient().psFailedReport(failedCounters);
Thread.sleep(20000);
for (int i = iterNum; i < 2 * iterNum; i++) {
DenseIntVector update = new DenseIntVector(dim);
for (int j = 0; j < dim; j++) {
update.set(j, 1);
}
update.setMatrixId(matrixClient.getMatrixId());
update.setRowId(0);
matrixClient.increment(update);
matrixClient.clock().get();
Thread.sleep(1000);
MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps1w1.getPartition(0));
assertNotNull(ps1w1.getPartition(1));
IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
int part0Size = ps1w1.getRow(0, 0).size();
IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
int part1Size = ps1w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
}
ParameterServer ps2Attempt = LocalClusterContext.get().getPS(ps2Attempt1Id).getPS();
for (int i = iterNum * 2; i < 3 * iterNum; i++) {
DenseIntVector update = new DenseIntVector(dim);
for (int j = 0; j < dim; j++) {
update.set(j, 1);
}
update.setMatrixId(matrixClient.getMatrixId());
update.setRowId(0);
matrixClient.increment(update);
matrixClient.clock().get();
Thread.sleep(1000);
MatrixStorageManager ps1Storage = ps1Attempt0.getMatrixStorageManager();
ServerMatrix ps1w1 = ps1Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps1w1.getPartition(0));
assertNotNull(ps1w1.getPartition(1));
IntBuffer row0Part0 = ((ServerDenseIntRow) ps1w1.getRow(0, 0)).getData();
int part0Size = ps1w1.getRow(0, 0).size();
IntBuffer row0Part1 = ((ServerDenseIntRow) ps1w1.getRow(1, 0)).getData();
int part1Size = ps1w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
MatrixStorageManager ps2Storage = ps2Attempt.getMatrixStorageManager();
ServerMatrix ps2w1 = ps2Storage.getMatrix(matrixClient.getMatrixId());
assertNotNull(ps2w1.getPartition(0));
assertNotNull(ps2w1.getPartition(1));
row0Part0 = ((ServerDenseIntRow) ps2w1.getRow(0, 0)).getData();
part0Size = ps2w1.getRow(0, 0).size();
row0Part1 = ((ServerDenseIntRow) ps2w1.getRow(1, 0)).getData();
part1Size = ps2w1.getRow(1, 0).size();
assertEquals(sum(row0Part0, part0Size), (i + 1) * dim / 2);
assertEquals(sum(row0Part1, part1Size), (i + 1) * dim / 2);
}
}
use of com.tencent.angel.ps.impl.matrix.ServerMatrix in project angel by Tencent.
the class PSManagerTest method testPSReport.
@Test
public void testPSReport() throws Exception {
try {
ParameterServer ps = LocalClusterContext.get().getPS(psAttempt0Id).getPS();
Location masterLoc = ps.getMasterLocation();
TConnection connection = TConnectionManager.getConnection(ps.getConf());
MasterProtocol master = connection.getMasterService(masterLoc.getIp(), masterLoc.getPort());
PSReportRequest.Builder builder = PSReportRequest.newBuilder();
builder.setPsAttemptId(ProtobufUtil.convertToIdProto(psAttempt0Id));
Pair.Builder pairBuilder = Pair.newBuilder();
pairBuilder.setKey("ps_key1");
pairBuilder.setValue("100");
builder.addMetrics(pairBuilder.build());
pairBuilder.setKey("ps_key2");
pairBuilder.setValue("200");
builder.addMetrics(pairBuilder.build());
builder.setPsFailedReports(MLProtos.PSFailedReportsProto.getDefaultInstance());
MatrixReportProto.Builder matrixBuilder = MatrixReportProto.newBuilder();
ConcurrentHashMap<Integer, ServerMatrix> matrixIdMap = ps.getMatrixStorageManager().getMatrices();
for (Entry<Integer, ServerMatrix> matrixEntry : matrixIdMap.entrySet()) {
builder.addMatrixReports((matrixBuilder.setMatrixId(matrixEntry.getKey()).setMatrixName(matrixEntry.getValue().getName())));
}
PSReportResponse response = master.psReport(null, builder.build());
assertEquals(response.getPsCommand(), PSCommandProto.PSCOMMAND_OK);
assertEquals(response.getNeedCreateMatricesCount(), 0);
assertEquals(response.getNeedReleaseMatrixIdsCount(), 0);
AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
ParameterServerManager psManager = angelAppMaster.getAppContext().getParameterServerManager();
AMParameterServer amPs = psManager.getParameterServer(psId);
PSAttempt psAttempt = amPs.getPSAttempt(psAttempt0Id);
Map<String, String> metrices = psAttempt.getMetrices();
assertTrue(metrices.get("ps_key1").equals("100"));
assertTrue(metrices.get("ps_key2").equals("200"));
PSAttemptId psAttempt1Id = new PSAttemptId(psId, 1);
builder.setPsAttemptId(ProtobufUtil.convertToIdProto(psAttempt1Id));
response = master.psReport(null, builder.build());
assertEquals(response.getPsCommand(), PSCommandProto.PSCOMMAND_SHUTDOWN);
} catch (Exception x) {
LOG.error("run testPSReport failed ", x);
throw x;
}
}
Aggregations