Examples with TezClient org.apache.tez.client.TezClient used on opensource projects

Example 21 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestMRRJobsDAGApi method testMRRSleepJobDagSubmitCore.

public State testMRRSleepJobDagSubmitCore(boolean dagViaRPC, boolean killDagWhileRunning, boolean closeSessionBeforeSubmit, TezClient reUseTezSession, boolean genSplitsInAM, Class<? extends InputInitializer> initializerClass, Map<String, LocalResource> additionalLocalResources) throws IOException, InterruptedException, TezException, ClassNotFoundException, YarnException {
    LOG.info("\n\n\nStarting testMRRSleepJobDagSubmit().");
    JobConf stage1Conf = new JobConf(mrrTezCluster.getConfig());
    JobConf stage2Conf = new JobConf(mrrTezCluster.getConfig());
    JobConf stage3Conf = new JobConf(mrrTezCluster.getConfig());
    stage1Conf.setLong(MRRSleepJob.MAP_SLEEP_TIME, 1);
    stage1Conf.setInt(MRRSleepJob.MAP_SLEEP_COUNT, 1);
    stage1Conf.setInt(MRJobConfig.NUM_MAPS, 1);
    stage1Conf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName());
    stage1Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
    stage1Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());
    stage1Conf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR, SleepInputFormat.class.getName());
    stage1Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());
    stage2Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1);
    stage2Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1);
    stage2Conf.setInt(MRJobConfig.NUM_REDUCES, 1);
    stage2Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName());
    stage2Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
    stage2Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());
    stage2Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());
    stage3Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1);
    stage3Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1);
    stage3Conf.setInt(MRJobConfig.NUM_REDUCES, 1);
    stage3Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName());
    stage3Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
    stage3Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());
    MRHelpers.translateMRConfToTez(stage1Conf);
    MRHelpers.translateMRConfToTez(stage2Conf);
    MRHelpers.translateMRConfToTez(stage3Conf);
    MRHelpers.configureMRApiUsage(stage1Conf);
    MRHelpers.configureMRApiUsage(stage2Conf);
    MRHelpers.configureMRApiUsage(stage3Conf);
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf);
    UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf);
    DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000));
    Class<? extends InputInitializer> inputInitializerClazz = genSplitsInAM ? (initializerClass == null ? MRInputAMSplitGenerator.class : initializerClass) : null;
    LOG.info("Using initializer class: " + initializerClass);
    DataSourceDescriptor dsd;
    if (!genSplitsInAM) {
        dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, remoteStagingDir, true);
    } else {
        if (initializerClass == null) {
            dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class).build();
        } else {
            InputInitializerDescriptor iid = InputInitializerDescriptor.create(inputInitializerClazz.getName());
            dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class).setCustomInitializerDescriptor(iid).build();
        }
    }
    Vertex stage1Vertex = Vertex.create("map", ProcessorDescriptor.create(MapProcessor.class.getName()).setUserPayload(stage1Payload), dsd.getNumberOfShards(), Resource.newInstance(256, 1));
    stage1Vertex.addDataSource("MRInput", dsd);
    Vertex stage2Vertex = Vertex.create("ireduce", ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage2Payload), 1, Resource.newInstance(256, 1));
    Vertex stage3Vertex = Vertex.create("reduce", ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage3Payload), 1, Resource.newInstance(256, 1));
    stage3Conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, true);
    DataSinkDescriptor dataSinkDescriptor = MROutputLegacy.createConfigBuilder(stage3Conf, NullOutputFormat.class).build();
    Assert.assertFalse(dataSinkDescriptor.getOutputDescriptor().getHistoryText().isEmpty());
    stage3Vertex.addDataSink("MROutput", dataSinkDescriptor);
    // TODO env, resources
    dag.addVertex(stage1Vertex);
    dag.addVertex(stage2Vertex);
    dag.addVertex(stage3Vertex);
    Edge edge1 = Edge.create(stage1Vertex, stage2Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage2Payload), InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage2Payload)));
    Edge edge2 = Edge.create(stage2Vertex, stage3Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage3Payload), InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload)));
    dag.addEdge(edge1);
    dag.addEdge(edge2);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    DAGClient dagClient = null;
    boolean reuseSession = reUseTezSession != null;
    TezClient tezSession = null;
    if (!dagViaRPC) {
        Preconditions.checkArgument(reuseSession == false);
    }
    if (!reuseSession) {
        TezConfiguration tempTezconf = new TezConfiguration(tezConf);
        if (!dagViaRPC) {
            tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
        } else {
            tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
        }
        tezSession = TezClient.create("testsession", tempTezconf);
        tezSession.start();
    } else {
        tezSession = reUseTezSession;
    }
    if (!dagViaRPC) {
        // TODO Use utility method post TEZ-205 to figure out AM arguments etc.
        dagClient = tezSession.submitDAG(dag);
    }
    if (dagViaRPC && closeSessionBeforeSubmit) {
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();
        boolean sentKillSession = false;
        while (true) {
            Thread.sleep(500l);
            ApplicationReport appReport = yarnClient.getApplicationReport(tezSession.getAppMasterApplicationId());
            if (appReport == null) {
                continue;
            }
            YarnApplicationState appState = appReport.getYarnApplicationState();
            if (!sentKillSession) {
                if (appState == YarnApplicationState.RUNNING) {
                    tezSession.stop();
                    sentKillSession = true;
                }
            } else {
                if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED) {
                    LOG.info("Application completed after sending session shutdown" + ", yarnApplicationState=" + appState + ", finalAppStatus=" + appReport.getFinalApplicationStatus());
                    Assert.assertEquals(YarnApplicationState.FINISHED, appState);
                    Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, appReport.getFinalApplicationStatus());
                    break;
                }
            }
        }
        yarnClient.stop();
        return null;
    }
    if (dagViaRPC) {
        LOG.info("Submitting dag to tez session with appId=" + tezSession.getAppMasterApplicationId() + " and Dag Name=" + dag.getName());
        if (additionalLocalResources != null) {
            tezSession.addAppMasterLocalFiles(additionalLocalResources);
        }
        dagClient = tezSession.submitDAG(dag);
        Assert.assertEquals(TezAppMasterStatus.RUNNING, tezSession.getAppMasterStatus());
    }
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        if (killDagWhileRunning && dagStatus.getState() == DAGStatus.State.RUNNING) {
            LOG.info("Killing running dag/session");
            if (dagViaRPC) {
                tezSession.stop();
            } else {
                dagClient.tryKillDAG();
            }
        }
        dagStatus = dagClient.getDAGStatus(null);
    }
    if (!reuseSession) {
        tezSession.stop();
    }
    return dagStatus.getState();
}

Example 22 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class UnionExample method run.

public boolean run(String inputPath, String outputPath, Configuration conf) throws Exception {
    System.out.println("Running UnionExample");
    // conf and UGI
    TezConfiguration tezConf;
    if (conf != null) {
        tezConf = new TezConfiguration(conf);
    } else {
        tezConf = new TezConfiguration();
    }
    UserGroupInformation.setConfiguration(tezConf);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    // staging dir
    FileSystem fs = FileSystem.get(tezConf);
    String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR + user + Path.SEPARATOR + ".staging" + Path.SEPARATOR + Path.SEPARATOR + Long.toString(System.currentTimeMillis());
    Path stagingDir = new Path(stagingDirStr);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
    stagingDir = fs.makeQualified(stagingDir);
    // No need to add jar containing this class as assumed to be part of
    // the tez jars.
    // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
    // is the same filesystem as the one used for Input/Output.
    TezClient tezSession = TezClient.create("UnionExampleSession", tezConf);
    tezSession.start();
    DAGClient dagClient = null;
    try {
        if (fs.exists(new Path(outputPath))) {
            throw new FileAlreadyExistsException("Output directory " + outputPath + " already exists");
        }
        Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>();
        DAG dag = createDAG(fs, tezConf, localResources, stagingDir, inputPath, outputPath);
        tezSession.waitTillReady();
        dagClient = tezSession.submitDAG(dag);
        // monitoring
        DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(EnumSet.of(StatusGetOpts.GET_COUNTERS));
        if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
            System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
            return false;
        }
        return true;
    } finally {
        fs.delete(stagingDir, true);
        tezSession.stop();
    }
}

Example 23 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestTezJobs method testAMClientHeartbeatTimeout.

@Test(timeout = 60000)
public void testAMClientHeartbeatTimeout() throws Exception {
    Path stagingDirPath = new Path("/tmp/timeout-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    YarnClient yarnClient = YarnClient.createYarnClient();
    try {
        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();
        List<ApplicationReport> apps = yarnClient.getApplications();
        int appsBeforeCount = apps != null ? apps.size() : 0;
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
        tezConf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS, 5);
        TezClient tezClient = TezClient.create("testAMClientHeartbeatTimeout", tezConf, true);
        tezClient.start();
        tezClient.cancelAMKeepAlive(true);
        ApplicationId appId = tezClient.getAppMasterApplicationId();
        apps = yarnClient.getApplications();
        int appsAfterCount = apps != null ? apps.size() : 0;
        // Running in session mode. So should only create 1 more app.
        Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
        ApplicationReport report;
        while (true) {
            report = yarnClient.getApplicationReport(appId);
            if (report.getYarnApplicationState() == YarnApplicationState.FINISHED || report.getYarnApplicationState() == YarnApplicationState.FAILED || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
                break;
            }
            Thread.sleep(1000);
        }
        // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
        Thread.sleep(2000);
        report = yarnClient.getApplicationReport(appId);
        LOG.info("App Report for appId=" + appId + ", report=" + report);
        Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(), report.getDiagnostics().contains("Client-to-AM Heartbeat timeout interval expired"));
    } finally {
        remoteFs.delete(stagingDirPath, true);
        if (yarnClient != null) {
            yarnClient.stop();
        }
    }
}

Example 24 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestTezJobs method testSortMergeJoinExamplePipeline.

/**
 * test whole {@link SortMergeJoinExample} pipeline as following: <br>
 * {@link JoinDataGen} -> {@link SortMergeJoinExample} -> {@link JoinValidate}
 * @throws Exception
 */
@Test(timeout = 120000)
public void testSortMergeJoinExamplePipeline() throws Exception {
    Path testDir = new Path("/tmp/testSortMergeExample");
    Path stagingDirPath = new Path("/tmp/tez-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    remoteFs.mkdirs(testDir);
    Path dataPath1 = new Path(testDir, "inPath1");
    Path dataPath2 = new Path(testDir, "inPath2");
    Path expectedOutputPath = new Path(testDir, "expectedOutputPath");
    Path outPath = new Path(testDir, "outPath");
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    TezClient tezSession = null;
    try {
        tezSession = TezClient.create("SortMergeExampleSession", tezConf, true);
        tezSession.start();
        JoinDataGen dataGen = new JoinDataGen();
        String[] dataGenArgs = new String[] { dataPath1.toString(), "1048576", dataPath2.toString(), "524288", expectedOutputPath.toString(), "2" };
        assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezSession));
        SortMergeJoinExample joinExample = new SortMergeJoinExample();
        String[] args = new String[] { dataPath1.toString(), dataPath2.toString(), "2", outPath.toString() };
        assertEquals(0, joinExample.run(tezConf, args, tezSession));
        JoinValidate joinValidate = new JoinValidate();
        String[] validateArgs = new String[] { expectedOutputPath.toString(), outPath.toString(), "3" };
        assertEquals(0, joinValidate.run(tezConf, validateArgs, tezSession));
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 25 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestTezJobs method testSessionTimeout.

@Test(timeout = 60000)
public void testSessionTimeout() throws Exception {
    Path stagingDirPath = new Path("/tmp/sessiontimeout-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    YarnClient yarnClient = YarnClient.createYarnClient();
    try {
        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();
        List<ApplicationReport> apps = yarnClient.getApplications();
        int appsBeforeCount = apps != null ? apps.size() : 0;
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
        tezConf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 5);
        TezClient tezClient = TezClient.create("testSessionTimeout", tezConf, true);
        tezClient.start();
        ApplicationId appId = tezClient.getAppMasterApplicationId();
        apps = yarnClient.getApplications();
        int appsAfterCount = apps != null ? apps.size() : 0;
        // Running in session mode. So should only create 1 more app.
        Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
        ApplicationReport report;
        while (true) {
            report = yarnClient.getApplicationReport(appId);
            if (report.getYarnApplicationState() == YarnApplicationState.FINISHED || report.getYarnApplicationState() == YarnApplicationState.FAILED || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
                break;
            }
            Thread.sleep(1000);
        }
        // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
        Thread.sleep(2000);
        report = yarnClient.getApplicationReport(appId);
        LOG.info("App Report for appId=" + appId + ", report=" + report);
        Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(), report.getDiagnostics().contains("Session timed out"));
    } finally {
        remoteFs.delete(stagingDirPath, true);
        if (yarnClient != null) {
            yarnClient.stop();
        }
    }
}