Examples with SessionNotReady org.apache.tez.dag.api.SessionNotReady used on opensource projects

Example 1 with SessionNotReady

use of org.apache.tez.dag.api.SessionNotReady in project tez by apache.

the class TezClient method preWarm.

/**
 * API to help pre-allocate containers in session mode. In non-session mode
 * this is ignored. The pre-allocated containers may be re-used by subsequent
 * job DAGs to improve performance.
 * The preWarm vertex should be configured and setup exactly
 * like the other vertices in the job DAGs so that the pre-allocated
 * containers may be re-used by the subsequent DAGs to improve performance.
 * The processor for the preWarmVertex may be used to pre-warm the containers
 * by pre-loading classes etc. It should be short-running so that pre-warming
 * does not block real execution. Users can specify their custom processors or
 * use the PreWarmProcessor from the runtime library.
 * The parallelism of the preWarmVertex will determine the number of preWarmed
 * containers.
 * Pre-warming is best efforts and among other factors is limited by the free
 * resources on the cluster. Based on the specified timeout value it returns
 * false if the status is not READY after the wait period.
 * @param preWarmVertex
 * @param timeout
 * @param unit
 * @throws TezException
 * @throws IOException
 */
@Unstable
public synchronized void preWarm(PreWarmVertex preWarmVertex, long timeout, TimeUnit unit) throws TezException, IOException {
    if (!isSession) {
        // do nothing for non session mode. This is there to let the code
        // work correctly in both modes
        LOG.warn("preWarm is not supported in non-session mode," + "please use session-mode of TezClient");
        return;
    }
    verifySessionStateForSubmission();
    DAG dag = org.apache.tez.dag.api.DAG.create(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX + "_" + preWarmDAGCounter++);
    dag.addVertex(preWarmVertex);
    boolean isReady;
    try {
        isReady = waitTillReady(timeout, unit);
    } catch (InterruptedException e) {
        throw new IOException("Interrupted while waiting for AM to become " + "available", e);
    }
    if (isReady) {
        submitDAG(dag);
    } else {
        throw new SessionNotReady("Tez AM not ready, could not submit DAG");
    }
}

Example 2 with SessionNotReady

use of org.apache.tez.dag.api.SessionNotReady in project tez by apache.

the class TestTezClient method testPreWarmWithTimeout.

@Test(timeout = 30000)
public void testPreWarmWithTimeout() throws Exception {
    long startTime = 0, endTime = 0;
    TezClientForTest client = configureAndCreateTezClient();
    final TezClientForTest spyClient = spy(client);
    doCallRealMethod().when(spyClient).start();
    doCallRealMethod().when(spyClient).stop();
    spyClient.start();
    when(spyClient.mockYarnClient.getApplicationReport(spyClient.mockAppId).getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING);
    when(spyClient.sessionAmProxy.getAMStatus((RpcController) any(), (GetAMStatusRequestProto) any())).thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.INITIALIZING).build());
    PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1));
    int timeout = 5000;
    try {
        startTime = Time.monotonicNow();
        spyClient.preWarm(vertex, timeout, TimeUnit.MILLISECONDS);
        fail("PreWarm should have encountered an Exception!");
    } catch (SessionNotReady te) {
        endTime = Time.monotonicNow();
        assertTrue("Time taken is not as expected", (endTime - startTime) > timeout);
        verify(spyClient, times(0)).submitDAG(any(DAG.class));
        Assert.assertTrue("Unexpected Exception message", te.getMessage().contains("Tez AM not ready"));
    }
    when(spyClient.sessionAmProxy.getAMStatus((RpcController) any(), (GetAMStatusRequestProto) any())).thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.READY).build());
    try {
        startTime = Time.monotonicNow();
        spyClient.preWarm(vertex, timeout, TimeUnit.MILLISECONDS);
        endTime = Time.monotonicNow();
        assertTrue("Time taken is not as expected", (endTime - startTime) <= timeout);
        verify(spyClient, times(1)).submitDAG(any(DAG.class));
    } catch (TezException te) {
        fail("PreWarm should have succeeded!");
    }
    Thread amStateThread = new Thread() {

        @Override
        public void run() {
            CountDownLatch latch = new CountDownLatch(1);
            try {
                when(spyClient.sessionAmProxy.getAMStatus((RpcController) any(), (GetAMStatusRequestProto) any())).thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.INITIALIZING).build());
                latch.await(1000, TimeUnit.MILLISECONDS);
                when(spyClient.sessionAmProxy.getAMStatus((RpcController) any(), (GetAMStatusRequestProto) any())).thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.READY).build());
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ServiceException e) {
                e.printStackTrace();
            }
        }
    };
    amStateThread.start();
    startTime = Time.monotonicNow();
    spyClient.preWarm(vertex, timeout, TimeUnit.MILLISECONDS);
    endTime = Time.monotonicNow();
    assertTrue("Time taken is not as expected", (endTime - startTime) <= timeout);
    verify(spyClient, times(2)).submitDAG(any(DAG.class));
    spyClient.stop();
    client.stop();
}