Examples with SubmitDAGRequestProto org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto used on opensource projects

Example 1 with SubmitDAGRequestProto

use of org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto in project tez by apache.

the class TestTezClient method testTezClient.

public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) throws Exception {
    Map<String, LocalResource> lrs = Maps.newHashMap();
    String lrName1 = "LR1";
    lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null);
    ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
    when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING);
    client.start();
    verify(client.mockYarnClient, times(1)).init((Configuration) any());
    verify(client.mockYarnClient, times(1)).start();
    if (isSession) {
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        ApplicationSubmissionContext context = captor.getValue();
        Assert.assertEquals(3, context.getAMContainerSpec().getLocalResources().size());
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1));
    } else {
        verify(client.mockYarnClient, times(0)).submitApplication(captor.capture());
    }
    String mockLR1Name = "LR1";
    Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1));
    DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG);
    DAGClient dagClient = client.submitDAG(dag);
    assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
    assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString());
    if (isSession) {
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        verify(client.sessionAmProxy, times(1)).submitDAG((RpcController) any(), (SubmitDAGRequestProto) any());
    } else {
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        ApplicationSubmissionContext context = captor.getValue();
        Assert.assertEquals(4, context.getAMContainerSpec().getLocalResources().size());
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1));
    }
    // add resources
    String lrName2 = "LR2";
    lrs.clear();
    lrs.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    client.addAppMasterLocalFiles(lrs);
    ApplicationId appId2 = ApplicationId.newInstance(0, 2);
    when(client.mockYarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId2);
    when(client.mockYarnClient.getApplicationReport(appId2).getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING);
    dag = DAG.create("DAG").addVertex(Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)));
    dagClient = client.submitDAG(dag);
    if (isSession) {
        // same app master
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
        assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString());
        // additional resource is sent
        ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
        verify(client.sessionAmProxy, times(2)).submitDAG((RpcController) any(), captor1.capture());
        SubmitDAGRequestProto proto = captor1.getValue();
        Assert.assertEquals(1, proto.getAdditionalAmResources().getLocalResourcesCount());
        Assert.assertEquals(lrName2, proto.getAdditionalAmResources().getLocalResources(0).getName());
    } else {
        // new app master
        assertTrue(dagClient.getExecutionContext().contains(appId2.toString()));
        assertEquals(dagClient.getSessionIdentifierString(), appId2.toString());
        verify(client.mockYarnClient, times(2)).submitApplication(captor.capture());
        // additional resource is added
        ApplicationSubmissionContext context = captor.getValue();
        Assert.assertEquals(5, context.getAMContainerSpec().getLocalResources().size());
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1));
        assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName2));
    }
    if (shouldStop) {
        client.stop();
        if (isSession) {
            verify(client.sessionAmProxy, times(1)).shutdownSession((RpcController) any(), (ShutdownSessionRequestProto) any());
        }
        verify(client.mockYarnClient, times(1)).stop();
    }
    return client;
}

Example 2 with SubmitDAGRequestProto

use of org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto in project tez by apache.

the class TestTezClient method testPreWarm.

@Test(timeout = 5000)
public void testPreWarm() throws Exception {
    TezClientForTest client = configureAndCreateTezClient();
    client.start();
    when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING);
    when(client.sessionAmProxy.getAMStatus((RpcController) any(), (GetAMStatusRequestProto) any())).thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.READY).build());
    PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1));
    client.preWarm(vertex);
    ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
    verify(client.sessionAmProxy, times(1)).submitDAG((RpcController) any(), captor1.capture());
    SubmitDAGRequestProto proto = captor1.getValue();
    assertTrue(proto.getDAGPlan().getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX));
    client.stop();
}

Example 3 with SubmitDAGRequestProto

use of org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto in project tez by apache.

the class TezClient method submitDAGSession.

private DAGClient submitDAGSession(DAG dag) throws TezException, IOException {
    Preconditions.checkState(isSession == true, "submitDAG with additional resources applies to only session mode. " + "In non-session mode please specify all resources in the initial configuration");
    verifySessionStateForSubmission();
    String dagId = null;
    String callerContextStr = "";
    if (dag.getCallerContext() != null) {
        callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString();
    }
    LOG.info("Submitting dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId + ", dagName=" + dag.getName() + callerContextStr);
    if (!additionalLocalResources.isEmpty()) {
        for (LocalResource lr : additionalLocalResources.values()) {
            Preconditions.checkArgument(lr.getType() == LocalResourceType.FILE, "LocalResourceType: " + lr.getType() + " is not supported, only " + LocalResourceType.FILE + " is supported");
        }
    }
    Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
    DAGPlan dagPlan = TezClientUtils.prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, usingTezArchiveDeploy, sessionCredentials, servicePluginsDescriptor, javaOptsChecker);
    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder();
    requestBuilder.setDAGPlan(dagPlan);
    if (!additionalLocalResources.isEmpty()) {
        requestBuilder.setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(additionalLocalResources));
    }
    additionalLocalResources.clear();
    // if request size exceeds maxSubmitDAGRequestSizeThroughIPC, we serialize them to HDFS
    SubmitDAGRequestProto request = requestBuilder.build();
    if (request.getSerializedSize() > maxSubmitDAGRequestSizeThroughIPC) {
        Path dagPlanPath = new Path(TezCommonUtils.getTezSystemStagingPath(amConfig.getTezConfiguration(), sessionAppId.toString()), TezConstants.TEZ_PB_PLAN_BINARY_NAME + serializedSubmitDAGPlanRequestCounter.incrementAndGet());
        try (FSDataOutputStream fsDataOutputStream = stagingFs.create(dagPlanPath, false)) {
            LOG.info("Send dag plan using YARN local resources since it's too large" + ", dag plan size=" + request.getSerializedSize() + ", max dag plan size through IPC=" + maxSubmitDAGRequestSizeThroughIPC + ", max IPC message size= " + amConfig.getTezConfiguration().getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT));
            request.writeTo(fsDataOutputStream);
            request = requestBuilder.clear().setSerializedRequestPath(stagingFs.resolvePath(dagPlanPath).toString()).build();
        }
    }
    DAGClientAMProtocolBlockingPB proxy = null;
    try {
        proxy = waitForProxy();
    } catch (InterruptedException e) {
        throw new IOException("Interrupted while trying to create a connection to the AM", e);
    }
    if (proxy == null) {
        try {
            LOG.warn("DAG submission to session timed out, stopping session");
            stop();
        } catch (Throwable t) {
            LOG.info("Got an exception when trying to stop session", t);
        }
        throw new DAGSubmissionTimedOut("Could not submit DAG to Tez Session" + ", timed out after " + clientTimeout + " seconds");
    }
    try {
        SubmitDAGResponseProto response = proxy.submitDAG(null, request);
        // SubmitDAGResponseProto cannot be mocked
        if (response != null) {
            dagId = response.getDagId();
        }
    } catch (ServiceException e) {
        RPCUtil.unwrapAndThrowException(e);
    }
    LOG.info("Submitted dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId + ", dagId=" + dagId + ", dagName=" + dag.getName());
    return new DAGClientImpl(sessionAppId, dagId, amConfig.getTezConfiguration(), amConfig.getYarnConfiguration(), frameworkClient);
}

Example 4 with SubmitDAGRequestProto

use of org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto in project tez by apache.

the class TestTezClient method _testTezClientSessionLargeDAGPlan.

private void _testTezClientSessionLargeDAGPlan(int maxIPCMsgSize, int payloadSize, int amResourceSize, boolean shouldSerialize) throws Exception {
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);
    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, "target/" + this.getClass().getName());
    TezClientForTest client = configureAndCreateTezClient(null, true, conf);
    Map<String, LocalResource> localResourceMap = new HashMap<>();
    byte[] bytes = new byte[amResourceSize];
    Arrays.fill(bytes, (byte) 1);
    String lrName = new String(bytes);
    localResourceMap.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create("P");
    processorDescriptor.setUserPayload(UserPayload.create(ByteBuffer.allocate(payloadSize)));
    Vertex vertex = Vertex.create("Vertex", processorDescriptor, 1, Resource.newInstance(1, 1));
    DAG dag = DAG.create("DAG").addVertex(vertex);
    client.start();
    client.addAppMasterLocalFiles(localResourceMap);
    client.submitDAG(dag);
    client.stop();
    ArgumentCaptor<SubmitDAGRequestProto> captor = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
    verify(client.sessionAmProxy).submitDAG((RpcController) any(), captor.capture());
    SubmitDAGRequestProto request = captor.getValue();
    if (shouldSerialize) {
        /* we need manually delete the serialized dagplan since staging path here won't be destroyed */
        Path dagPlanPath = new Path(request.getSerializedRequestPath());
        FileSystem fs = FileSystem.getLocal(conf);
        fs.deleteOnExit(dagPlanPath);
        fs.delete(dagPlanPath, false);
        assertTrue(request.hasSerializedRequestPath());
        assertFalse(request.hasDAGPlan());
        assertFalse(request.hasAdditionalAmResources());
    } else {
        assertFalse(request.hasSerializedRequestPath());
        assertTrue(request.hasDAGPlan());
        assertTrue(request.hasAdditionalAmResources());
    }
}