Examples with TezClient org.apache.tez.client.TezClient used on opensource projects

Example 6 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestATSHistoryV15 method testATSLogLevelNone.

@Test
public void testATSLogLevelNone() throws Exception {
    TezClient tezSession = null;
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, "TEZ_DAG_ID");
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSV15HistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();
        applicationId = tezSession.getAppMasterApplicationId();
        dag.setHistoryLogLevel(HistoryLogLevel.NONE);
        DAGClient dagClient = tezSession.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
        // Verify HDFS data
        int count = verifyATSDataOnHDFS(atsActivePath, applicationId);
        Assert.assertEquals("Count is: " + count, 1, count);
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 7 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestATSHistoryV15 method testSimpleDAG.

@Test(timeout = 50000)
public void testSimpleDAG() throws Exception {
    TezClient tezSession = null;
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, "TEZ_DAG_ID");
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSV15HistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();
        applicationId = tezSession.getAppMasterApplicationId();
        DAGClient dagClient = tezSession.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
        // Verify HDFS data
        int count = verifyATSDataOnHDFS(atsActivePath, applicationId);
        Assert.assertEquals("Count is: " + count, 2, count);
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 8 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestShuffleHandlerJobs method testOrderedWordCount.

@Test(timeout = 300000)
public void testOrderedWordCount() throws Exception {
    String inputDirStr = "/tmp/owc-input/";
    Path inputDir = new Path(inputDirStr);
    Path stagingDirPath = new Path("/tmp/owc-staging-dir");
    remoteFs.mkdirs(inputDir);
    remoteFs.mkdirs(stagingDirPath);
    generateOrderedWordCountInput(inputDir, remoteFs);
    String outputDirStr = "/tmp/owc-output/";
    Path outputDir = new Path(outputDirStr);
    TezConfiguration tezConf = new TezConfiguration(tezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.set(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, ShuffleHandler.TEZ_SHUFFLE_SERVICEID);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION, true);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    tezConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, false);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
    TezClient tezSession = TezClient.create("WordCountTest", tezConf);
    tezSession.start();
    try {
        final OrderedWordCount job = new OrderedWordCount();
        Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[] { "-counter", inputDirStr, outputDirStr, "10" }, tezSession) == 0);
        verifyOutput(outputDir, remoteFs);
        tezSession.stop();
        ClientRMService rmService = tezCluster.getResourceManager().getClientRMService();
        boolean isAppComplete = false;
        while (!isAppComplete) {
            GetApplicationReportResponse resp = rmService.getApplicationReport(new GetApplicationReportRequest() {

                @Override
                public ApplicationId getApplicationId() {
                    return job.getAppId();
                }

                @Override
                public void setApplicationId(ApplicationId applicationId) {
                }
            });
            if (resp.getApplicationReport().getYarnApplicationState() == YarnApplicationState.FINISHED) {
                isAppComplete = true;
            }
            Thread.sleep(100);
        }
        for (int i = 0; i < NUM_NMS; i++) {
            String appPath = tezCluster.getTestWorkDir() + "/" + this.getClass().getName() + "-localDir-nm-" + i + "_0/usercache/" + UserGroupInformation.getCurrentUser().getUserName() + "/appcache/" + job.getAppId();
            String dagPathStr = appPath + "/dag_1";
            File fs = new File(dagPathStr);
            Assert.assertFalse(fs.exists());
            fs = new File(appPath);
            Assert.assertTrue(fs.exists());
        }
    } finally {
        remoteFs.delete(stagingDirPath, true);
    }
}

Example 9 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestATSHistoryWithACLs method testDisableSessionLogging.

/**
 * Test Disable Logging for all dags in a session
 * due to failure to create domain in session start
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDisableSessionLogging() throws Exception {
    TezClient tezSession = null;
    String viewAcls = "nobody nobody_group";
    SleepProcessorConfig spConf = new SleepProcessorConfig(1);
    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
    dag.addVertex(vertex);
    DAGAccessControls accessControls = new DAGAccessControls();
    accessControls.setUsersWithViewACLs(Collections.singleton("nobody2"));
    accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group2"));
    dag.setAccessControls(accessControls);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
    tezSession.start();
    // ////submit first dag //////
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        dagStatus = dagClient.getDAGStatus(null);
    }
    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    // ////submit second dag//////
    DAG dag2 = DAG.create("TezSleepProcessor2");
    vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
    dag2.addVertex(vertex);
    accessControls = new DAGAccessControls();
    accessControls.setUsersWithViewACLs(Collections.singleton("nobody3"));
    accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group3"));
    dag2.setAccessControls(accessControls);
    dagClient = tezSession.submitDAG(dag2);
    dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        dagStatus = dagClient.getDAGStatus(null);
    }
    tezSession.stop();
}

Example 10 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestExtServicesWithLocalMode method test1.

@Test(timeout = 30000)
public void test1() throws Exception {
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(confForJobs);
    TaskSchedulerDescriptor[] taskSchedulerDescriptors = new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskSchedulerService.class.getName()).setUserPayload(userPayload) };
    ContainerLauncherDescriptor[] containerLauncherDescriptors = new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceNoOpContainerLauncher.class.getName()).setUserPayload(userPayload) };
    TaskCommunicatorDescriptor[] taskCommunicatorDescriptors = new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskCommunicatorImpl.class.getName()).setUserPayload(userPayload) };
    ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, false, taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors);
    TezConfiguration tezConf = new TezConfiguration(confForJobs);
    TezClient tezClient = TezClient.newBuilder("test1", tezConf).setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
    try {
        tezClient.start();
        Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1");
        Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2");
        Path expectedResultPath = new Path(SRC_DATA_DIR, "expectedOutputPath");
        JoinDataGen dataGen = new JoinDataGen();
        String[] dataGenArgs = new String[] { dataPath1.toString(), "1048576", dataPath2.toString(), "524288", expectedResultPath.toString(), "2" };
        assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezClient));
        Path outputPath = new Path(SRC_DATA_DIR, "outPath");
        HashJoinExample joinExample = new HashJoinExample();
        String[] args = new String[] { dataPath1.toString(), dataPath2.toString(), "2", outputPath.toString() };
        assertEquals(0, joinExample.run(tezConf, args, tezClient));
        LOG.info("Completed generating Data - Expected Hash Result and Actual Join Result");
        assertEquals(0, tezTestServiceCluster.getNumSubmissions());
        // ext can consume from ext.
        runJoinValidate(tezClient, "allInExt", 7, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH);
        LOG.info("Completed allInExt");
        // uber can consume from uber.
        runJoinValidate(tezClient, "noneInExt", 0, null, null, null);
        LOG.info("Completed noneInExt");
        // uber can consume from ext
        runJoinValidate(tezClient, "lhsInExt", 2, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, null, null);
        LOG.info("Completed lhsInExt");
    // ext cannot consume from uber in this mode since there's no shuffle handler working,
    // and the local data transfer semantics may not match.
    } finally {
        tezClient.stop();
    }
}