Examples with TezClient org.apache.tez.client.TezClient used on opensource projects

Example 26 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestTezJobs method testHashJoinExamplePipeline.

/**
 * test whole {@link HashJoinExample} pipeline as following: <br>
 * {@link JoinDataGen} -> {@link HashJoinExample} -> {@link JoinValidate}
 * @throws Exception
 */
@Test(timeout = 120000)
public void testHashJoinExamplePipeline() throws Exception {
    Path testDir = new Path("/tmp/testHashJoinExample");
    Path stagingDirPath = new Path("/tmp/tez-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    remoteFs.mkdirs(testDir);
    Path dataPath1 = new Path(testDir, "inPath1");
    Path dataPath2 = new Path(testDir, "inPath2");
    Path expectedOutputPath = new Path(testDir, "expectedOutputPath");
    Path outPath = new Path(testDir, "outPath");
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    TezClient tezSession = null;
    try {
        tezSession = TezClient.create("HashJoinExampleSession", tezConf, true);
        tezSession.start();
        JoinDataGen dataGen = new JoinDataGen();
        String[] dataGenArgs = new String[] { "-counter", dataPath1.toString(), "1048576", dataPath2.toString(), "524288", expectedOutputPath.toString(), "2" };
        assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezSession));
        HashJoinExample joinExample = new HashJoinExample();
        String[] args = new String[] { dataPath1.toString(), dataPath2.toString(), "2", outPath.toString() };
        assertEquals(0, joinExample.run(tezConf, args, tezSession));
        JoinValidate joinValidate = new JoinValidate();
        String[] validateArgs = new String[] { "-counter", expectedOutputPath.toString(), outPath.toString(), "3" };
        assertEquals(0, joinValidate.run(tezConf, validateArgs, tezSession));
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 27 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestTezJobs method testInputInitializerEvents.

@Test(timeout = 60000)
public void testInputInitializerEvents() throws TezException, InterruptedException, IOException {
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    TezClient tezClient = TezClient.create("TestInputInitializerEvents", tezConf);
    tezClient.start();
    try {
        DAG dag = DAG.create("TestInputInitializerEvents");
        Vertex vertex1 = Vertex.create(VERTEX_WITH_INITIALIZER_NAME, ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1).addDataSource(INPUT1_NAME, DataSourceDescriptor.create(InputDescriptor.create(MultiAttemptDAG.NoOpInput.class.getName()), InputInitializerDescriptor.create(InputInitializerForTest.class.getName()), null));
        Vertex vertex2 = Vertex.create(EVENT_GENERATING_VERTEX_NAME, ProcessorDescriptor.create(InputInitializerEventGeneratingProcessor.class.getName()), 5);
        dag.addVertex(vertex1).addVertex(vertex2);
        DAGClient dagClient = tezClient.submitDAG(dag);
        dagClient.waitForCompletion();
        Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    } finally {
        tezClient.stop();
    }
}

Example 28 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestTaskErrorsUsingLocalMode method testNonFatalErrorReported.

@Test(timeout = 20000)
public void testNonFatalErrorReported() throws IOException, TezException, InterruptedException {
    TezClient tezClient = getTezClient("testNonFatalErrorReported");
    DAGClient dagClient = null;
    try {
        FailingProcessor.configureForNonFatalFail();
        DAG dag = DAG.create("testNonFatalErrorReported").addVertex(Vertex.create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1));
        dagClient = tezClient.submitDAG(dag);
        dagClient.waitForCompletion();
        assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
        assertEquals(4, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getFailedTaskAttemptCount());
    } finally {
        if (dagClient != null) {
            dagClient.close();
        }
        tezClient.stop();
    }
}

Example 29 with TezClient

use of org.apache.tez.client.TezClient in project hive by apache.

the class TezSessionState method isOpening.

public boolean isOpening() {
    if (session != null || sessionFuture == null) {
        return false;
    }
    try {
        TezClient session = sessionFuture.get(0, TimeUnit.NANOSECONDS);
        if (session == null) {
            return false;
        }
        this.session = session;
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        return false;
    } catch (ExecutionException e) {
        throw new RuntimeException(e);
    } catch (CancellationException e) {
        return false;
    } catch (TimeoutException e) {
        return true;
    }
    return false;
}

Example 30 with TezClient

use of org.apache.tez.client.TezClient in project hive by apache.

the class TezTask method ensureSessionHasResources.

/**
 * Ensures that the Tez Session is open and the AM has all necessary jars configured.
 */
@VisibleForTesting
void ensureSessionHasResources(TezSessionState session, String[] nonConfResources) throws Exception {
    TezClient client = session.getSession();
    // TODO null can also mean that this operation was interrupted. Should we really try to re-create the session in that case ?
    if (client == null) {
        // Note: the only sane case where this can happen is the non-pool one. We should get rid
        // of it, in non-pool case perf doesn't matter so we might as well open at get time
        // and then call update like we do in the else.
        // Can happen if the user sets the tez flag after the session was established.
        LOG.info("Tez session hasn't been created yet. Opening session");
        session.open(nonConfResources);
    } else {
        LOG.info("Session is already open");
        session.ensureLocalResources(conf, nonConfResources);
    }
}