Examples with PSMatricesSaveResult com.tencent.angel.model.PSMatricesSaveResult used on opensource projects

Example 1 with PSMatricesSaveResult

use of com.tencent.angel.model.PSMatricesSaveResult in project angel by Tencent.

the class MasterService method psReport.

/**
 * response for parameter server heartbeat
 *
 * @param controller rpc controller of protobuf
 * @param request heartbeat request
 */
@SuppressWarnings("unchecked")
@Override
public PSReportResponse psReport(RpcController controller, PSReportRequest request) throws ServiceException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("receive ps heartbeat request. request=" + request);
    }
    // parse parameter server counters
    List<Pair> params = request.getMetricsList();
    int size = params.size();
    Map<String, String> paramsMap = new HashMap<String, String>();
    for (int i = 0; i < size; i++) {
        paramsMap.put(params.get(i).getKey(), params.get(i).getValue());
    }
    PSAttemptId psAttemptId = ProtobufUtil.convertToId(request.getPsAttemptId());
    PSReportResponse.Builder resBuilder = PSReportResponse.newBuilder();
    if (!context.getParameterServerManager().isAlive(psAttemptId)) {
        // if psAttemptId is not in monitor set, just return a PSCOMMAND_SHUTDOWN command.
        LOG.error("ps attempt " + psAttemptId + " is not in running ps attempt set");
        resBuilder.setPsCommand(PSCommandProto.PSCOMMAND_SHUTDOWN);
    } else {
        resBuilder.setPsCommand(PSCommandProto.PSCOMMAND_OK);
        // refresh last heartbeat timestamp
        context.getParameterServerManager().alive(psAttemptId);
        // send a state update event to the specific PSAttempt
        context.getEventHandler().handle(new PSAttemptStateUpdateEvent(psAttemptId, paramsMap));
        // Check is there save request
        PSMatricesSaveContext subSaveContext = context.getModelSaver().getSaveContext(psAttemptId.getPsId());
        PSMatricesSaveResult subSaveResult = context.getModelSaver().getSaveResult(psAttemptId.getPsId());
        if (subSaveContext != null && subSaveResult != null && (subSaveContext.getRequestId() == subSaveResult.getRequestId()) && (subSaveResult.getState() == SaveState.INIT || subSaveResult.getState() == SaveState.SAVING)) {
            // LOG.info("PS " + psAttemptId + " need save " + subSaveContext);
            resBuilder.setNeedSaveMatrices(ProtobufUtil.convert(subSaveContext));
        }
        // Check is there load request
        PSMatricesLoadContext subLoadContext = context.getModelLoader().getLoadContext(psAttemptId.getPsId());
        PSMatricesLoadResult subLoadResult = context.getModelLoader().getLoadResult(psAttemptId.getPsId());
        if (subLoadContext != null && subLoadResult != null && subLoadContext.getRequestId() == subLoadResult.getRequestId() && (subLoadResult.getState() == LoadState.INIT || subLoadResult.getState() == LoadState.LOADING)) {
            // LOG.info("PS " + psAttemptId + " need load " + subLoadContext);
            resBuilder.setNeedLoadMatrices(ProtobufUtil.convert(subLoadContext));
        }
        // check matrix metadata inconsistencies between master and parameter server.
        // if a matrix exists on the Master and does not exist on ps, then it is necessary to notify ps to establish the matrix
        // if a matrix exists on the ps and does not exist on master, then it is necessary to notify ps to remove the matrix
        List<MatrixReportProto> matrixReportsProto = request.getMatrixReportsList();
        List<Integer> needReleaseMatrices = new ArrayList<>();
        List<MatrixMeta> needCreateMatrices = new ArrayList<>();
        List<RecoverPartKey> needRecoverParts = new ArrayList<>();
        List<MatrixReport> matrixReports = ProtobufUtil.convertToMatrixReports(matrixReportsProto);
        context.getMatrixMetaManager().syncMatrixInfos(matrixReports, needCreateMatrices, needReleaseMatrices, needRecoverParts, psAttemptId.getPsId());
        size = needCreateMatrices.size();
        for (int i = 0; i < size; i++) {
            resBuilder.addNeedCreateMatrices(ProtobufUtil.convertToMatrixMetaProto(needCreateMatrices.get(i)));
        }
        size = needReleaseMatrices.size();
        for (int i = 0; i < size; i++) {
            resBuilder.addNeedReleaseMatrixIds(needReleaseMatrices.get(i));
        }
        size = needRecoverParts.size();
        for (int i = 0; i < size; i++) {
            resBuilder.addNeedRecoverParts(ProtobufUtil.convert(needRecoverParts.get(i)));
        }
    }
    return resBuilder.build();
}

Example 2 with PSMatricesSaveResult

use of com.tencent.angel.model.PSMatricesSaveResult in project angel by Tencent.

the class AMModelSaver method save.

private void save(ModelSaveRunningContext runningContext) {
    ModelSaveContext saveContext = runningContext.getSaveContext();
    try {
        lock.lock();
        currentRequestId = runningContext.getRequestId();
        LOG.info("Start to execute save request " + saveContext + " with request id=" + runningContext.getRequestId());
        // Split the user request to sub-requests to pss
        currentSubSaveContexts = split(currentRequestId, saveContext);
        subResults = new HashMap<>(currentSubSaveContexts.size());
        for (Map.Entry<ParameterServerId, PSMatricesSaveContext> entry : currentSubSaveContexts.entrySet()) {
            subResults.put(entry.getKey(), new PSMatricesSaveResult(entry.getValue().getRequestId(), entry.getValue().getSubRequestId(), SaveState.INIT));
        }
        receivedSubResult = 0;
    } finally {
        lock.unlock();
    }
}

Example 3 with PSMatricesSaveResult

use of com.tencent.angel.model.PSMatricesSaveResult in project angel by Tencent.

the class PSModelSaver method save.

/**
 * Save matrices
 *
 * @param saveContext save context
 */
public void save(PSMatricesSaveContext saveContext) {
    writeLock.lock();
    try {
        if (saveContexts.containsKey(saveContext.getRequestId())) {
            LOG.info("Save task " + saveContexts.get(saveContext.getRequestId()) + " is running");
            return;
        }
        if (currentRequestId != -1) {
            LOG.warn("There is another save task now, just stop it " + saveContexts.get(currentRequestId));
            if (fileOpExecutor != null) {
                fileOpExecutor.shutdown();
            }
        }
        currentRequestId = saveContext.getRequestId();
        lastRequestId = currentRequestId;
        saveContexts.put(currentRequestId, saveContext);
        results.put(currentRequestId, new PSMatricesSaveResult(currentRequestId, saveContext.getSubRequestId(), SaveState.SAVING));
        save(saveContext, results.get(currentRequestId));
    } finally {
        writeLock.unlock();
    }
/*
    try {
      writeLock.lock();

      if (saveContexts.containsKey(saveContext.getRequestId())) {
        LOG.info("Save task " + saveContexts.get(saveContext.getRequestId()) + " is running");
        return;
      }

      if (currentRequestId != -1) {
        LOG.warn(
          "There is another save task now, just stop it " + saveContexts.get(currentRequestId));
        if (fileOpExecutor != null) {
          fileOpExecutor.shutdown();
        }
      }
      currentRequestId = saveContext.getRequestId();
      lastRequestId = currentRequestId;
      saveContexts.put(currentRequestId, saveContext);
      results.put(currentRequestId,
        new PSMatricesSaveResult(currentRequestId, saveContext.getSubRequestId(),
          SaveState.SAVING));
      save(saveContext, results.get(currentRequestId));
    } finally {
      writeLock.unlock();
    }
    */
}

Example 4 with PSMatricesSaveResult

use of com.tencent.angel.model.PSMatricesSaveResult in project angel by Tencent.

the class AMModelSaver method psSaveStart.

/**
 * PS start saving
 *
 * @param psId PS id
 * @param requestId request id
 * @param subRequestId sub-request id
 */
public void psSaveStart(ParameterServerId psId, int requestId, int subRequestId) {
    try {
        lock.lock();
        if (currentRequestId == requestId) {
            PSMatricesSaveResult subResult = subResults.get(psId);
            subResult.setState(SaveState.SAVING);
        }
    } finally {
        lock.unlock();
    }
}