use of com.tencent.angel.model.PSMatricesLoadContext in project angel by Tencent.
the class MasterService method psReport.
/**
* response for parameter server heartbeat
*
* @param controller rpc controller of protobuf
* @param request heartbeat request
*/
@SuppressWarnings("unchecked")
@Override
public PSReportResponse psReport(RpcController controller, PSReportRequest request) throws ServiceException {
if (LOG.isDebugEnabled()) {
LOG.debug("receive ps heartbeat request. request=" + request);
}
// parse parameter server counters
List<Pair> params = request.getMetricsList();
int size = params.size();
Map<String, String> paramsMap = new HashMap<String, String>();
for (int i = 0; i < size; i++) {
paramsMap.put(params.get(i).getKey(), params.get(i).getValue());
}
PSAttemptId psAttemptId = ProtobufUtil.convertToId(request.getPsAttemptId());
PSReportResponse.Builder resBuilder = PSReportResponse.newBuilder();
if (!context.getParameterServerManager().isAlive(psAttemptId)) {
// if psAttemptId is not in monitor set, just return a PSCOMMAND_SHUTDOWN command.
LOG.error("ps attempt " + psAttemptId + " is not in running ps attempt set");
resBuilder.setPsCommand(PSCommandProto.PSCOMMAND_SHUTDOWN);
} else {
resBuilder.setPsCommand(PSCommandProto.PSCOMMAND_OK);
// refresh last heartbeat timestamp
context.getParameterServerManager().alive(psAttemptId);
// send a state update event to the specific PSAttempt
context.getEventHandler().handle(new PSAttemptStateUpdateEvent(psAttemptId, paramsMap));
// Check is there save request
PSMatricesSaveContext subSaveContext = context.getModelSaver().getSaveContext(psAttemptId.getPsId());
PSMatricesSaveResult subSaveResult = context.getModelSaver().getSaveResult(psAttemptId.getPsId());
if (subSaveContext != null && subSaveResult != null && (subSaveContext.getRequestId() == subSaveResult.getRequestId()) && (subSaveResult.getState() == SaveState.INIT || subSaveResult.getState() == SaveState.SAVING)) {
// LOG.info("PS " + psAttemptId + " need save " + subSaveContext);
resBuilder.setNeedSaveMatrices(ProtobufUtil.convert(subSaveContext));
}
// Check is there load request
PSMatricesLoadContext subLoadContext = context.getModelLoader().getLoadContext(psAttemptId.getPsId());
PSMatricesLoadResult subLoadResult = context.getModelLoader().getLoadResult(psAttemptId.getPsId());
if (subLoadContext != null && subLoadResult != null && subLoadContext.getRequestId() == subLoadResult.getRequestId() && (subLoadResult.getState() == LoadState.INIT || subLoadResult.getState() == LoadState.LOADING)) {
// LOG.info("PS " + psAttemptId + " need load " + subLoadContext);
resBuilder.setNeedLoadMatrices(ProtobufUtil.convert(subLoadContext));
}
// check matrix metadata inconsistencies between master and parameter server.
// if a matrix exists on the Master and does not exist on ps, then it is necessary to notify ps to establish the matrix
// if a matrix exists on the ps and does not exist on master, then it is necessary to notify ps to remove the matrix
List<MatrixReportProto> matrixReportsProto = request.getMatrixReportsList();
List<Integer> needReleaseMatrices = new ArrayList<>();
List<MatrixMeta> needCreateMatrices = new ArrayList<>();
List<RecoverPartKey> needRecoverParts = new ArrayList<>();
List<MatrixReport> matrixReports = ProtobufUtil.convertToMatrixReports(matrixReportsProto);
context.getMatrixMetaManager().syncMatrixInfos(matrixReports, needCreateMatrices, needReleaseMatrices, needRecoverParts, psAttemptId.getPsId());
size = needCreateMatrices.size();
for (int i = 0; i < size; i++) {
resBuilder.addNeedCreateMatrices(ProtobufUtil.convertToMatrixMetaProto(needCreateMatrices.get(i)));
}
size = needReleaseMatrices.size();
for (int i = 0; i < size; i++) {
resBuilder.addNeedReleaseMatrixIds(needReleaseMatrices.get(i));
}
size = needRecoverParts.size();
for (int i = 0; i < size; i++) {
resBuilder.addNeedRecoverParts(ProtobufUtil.convert(needRecoverParts.get(i)));
}
}
return resBuilder.build();
}
use of com.tencent.angel.model.PSMatricesLoadContext in project angel by Tencent.
the class ParameterServer method initMatricesData.
private void initMatricesData(final List<MatrixMeta> matrixMetas) throws IOException {
if (context.getPartReplication() > 1 && context.getPSAttemptId().getIndex() > 0) {
return;
}
// Recover PS from snapshot or load path
if (context.getPSAttemptId().getIndex() > 0) {
int matrixNum = matrixMetas.size();
List<PSMatrixLoadContext> matrixLoadContexts = new ArrayList<>(matrixMetas.size());
SnapshotRecover recover = new SnapshotRecover(context);
for (int i = 0; i < matrixNum; i++) {
// 1. First check old snapshot
Path inputPath = null;
try {
inputPath = recover.getSnapshotPath(matrixMetas.get(i).getId());
} catch (IOException e) {
LOG.error("Get snapshot path failed, ", e);
}
// 2. Check new checkpoints
if (inputPath == null) {
try {
List<SaveResult> saveResults = master.getCheckpoints(matrixMetas.get(i).getId());
if (saveResults == null || saveResults.isEmpty()) {
LOG.info("There is no checkpoint results for matrix " + matrixMetas.get(i).getName());
} else {
inputPath = new Path(saveResults.get(saveResults.size() - 1).getMatrixPath());
LOG.info("There is " + saveResults.size() + " checkpoint results for matrix + " + matrixMetas.get(i).getName() + " we choose the latest result in dir " + saveResults.get(saveResults.size() - 1).getMatrixPath());
}
} catch (ServiceException e) {
LOG.error("Get checkpoint results for matrix " + matrixMetas.get(i).getName() + " failed ", e);
}
}
// 3. Check load path setting and old save result
if (inputPath == null) {
try {
List<SaveResult> saveResults = master.getSaveResult(matrixMetas.get(i).getId());
if (saveResults == null || saveResults.isEmpty()) {
LOG.info("There is no old save result for matrix " + matrixMetas.get(i).getName());
} else {
inputPath = new Path(saveResults.get(saveResults.size() - 1).getMatrixPath());
LOG.info("There is " + saveResults.size() + " old save results for matrix + " + matrixMetas.get(i).getName() + " we choose the latest result in dir " + saveResults.get(saveResults.size() - 1).getMatrixPath());
}
} catch (ServiceException e) {
LOG.error("Get save results for matrix " + matrixMetas.get(i).getName() + " failed ", e);
}
}
if (inputPath != null) {
LOG.info("Load matrix " + matrixMetas.get(i).getName() + " from " + inputPath.toString());
matrixLoadContexts.add(new PSMatrixLoadContext(matrixMetas.get(i).getId(), inputPath.toString(), new ArrayList<>(matrixMetas.get(i).getPartitionMetas().keySet()), SnapshotFormat.class.getName()));
} else {
// Just init it again
if (matrixMetas.get(i).getInitFunc() != null) {
LOG.info("Matrix " + matrixMetas.get(i) + " has a init function " + matrixMetas.get(i).getInitFunc().getClass().getName() + ", use this function to reinit the matrix");
long startTs = System.currentTimeMillis();
matrixMetas.get(i).getInitFunc().init(context.getMatrixStorageManager().getMatrix(matrixMetas.get(i).getId()));
LOG.info("Reinit the matrix use time " + (System.currentTimeMillis() - startTs));
}
}
}
if (!matrixLoadContexts.isEmpty()) {
context.getIOExecutors().load(new PSMatricesLoadContext(-1, -1, matrixLoadContexts));
}
}
}
Aggregations