Examples with SubmitDAGResponseProto org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGResponseProto used on opensource projects

Example 1 with SubmitDAGResponseProto

use of org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGResponseProto in project tez by apache.

the class TezClient method submitDAGSession.

private DAGClient submitDAGSession(DAG dag) throws TezException, IOException {
    Preconditions.checkState(isSession == true, "submitDAG with additional resources applies to only session mode. " + "In non-session mode please specify all resources in the initial configuration");
    verifySessionStateForSubmission();
    String dagId = null;
    String callerContextStr = "";
    if (dag.getCallerContext() != null) {
        callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString();
    }
    LOG.info("Submitting dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId + ", dagName=" + dag.getName() + callerContextStr);
    if (!additionalLocalResources.isEmpty()) {
        for (LocalResource lr : additionalLocalResources.values()) {
            Preconditions.checkArgument(lr.getType() == LocalResourceType.FILE, "LocalResourceType: " + lr.getType() + " is not supported, only " + LocalResourceType.FILE + " is supported");
        }
    }
    Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
    DAGPlan dagPlan = TezClientUtils.prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, usingTezArchiveDeploy, sessionCredentials, servicePluginsDescriptor, javaOptsChecker);
    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder();
    requestBuilder.setDAGPlan(dagPlan);
    if (!additionalLocalResources.isEmpty()) {
        requestBuilder.setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(additionalLocalResources));
    }
    additionalLocalResources.clear();
    // if request size exceeds maxSubmitDAGRequestSizeThroughIPC, we serialize them to HDFS
    SubmitDAGRequestProto request = requestBuilder.build();
    if (request.getSerializedSize() > maxSubmitDAGRequestSizeThroughIPC) {
        Path dagPlanPath = new Path(TezCommonUtils.getTezSystemStagingPath(amConfig.getTezConfiguration(), sessionAppId.toString()), TezConstants.TEZ_PB_PLAN_BINARY_NAME + serializedSubmitDAGPlanRequestCounter.incrementAndGet());
        try (FSDataOutputStream fsDataOutputStream = stagingFs.create(dagPlanPath, false)) {
            LOG.info("Send dag plan using YARN local resources since it's too large" + ", dag plan size=" + request.getSerializedSize() + ", max dag plan size through IPC=" + maxSubmitDAGRequestSizeThroughIPC + ", max IPC message size= " + amConfig.getTezConfiguration().getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT));
            request.writeTo(fsDataOutputStream);
            request = requestBuilder.clear().setSerializedRequestPath(stagingFs.resolvePath(dagPlanPath).toString()).build();
        }
    }
    DAGClientAMProtocolBlockingPB proxy = null;
    try {
        proxy = waitForProxy();
    } catch (InterruptedException e) {
        throw new IOException("Interrupted while trying to create a connection to the AM", e);
    }
    if (proxy == null) {
        try {
            LOG.warn("DAG submission to session timed out, stopping session");
            stop();
        } catch (Throwable t) {
            LOG.info("Got an exception when trying to stop session", t);
        }
        throw new DAGSubmissionTimedOut("Could not submit DAG to Tez Session" + ", timed out after " + clientTimeout + " seconds");
    }
    try {
        SubmitDAGResponseProto response = proxy.submitDAG(null, request);
        // SubmitDAGResponseProto cannot be mocked
        if (response != null) {
            dagId = response.getDagId();
        }
    } catch (ServiceException e) {
        RPCUtil.unwrapAndThrowException(e);
    }
    LOG.info("Submitted dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId + ", dagId=" + dagId + ", dagName=" + dag.getName());
    return new DAGClientImpl(sessionAppId, dagId, amConfig.getTezConfiguration(), amConfig.getYarnConfiguration(), frameworkClient);
}