Examples with YarnApplicationState org.apache.hadoop.yarn.api.records.YarnApplicationState used on opensource projects

Example 6 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class TestMRTimelineEventHandling method testMRNewTimelineServiceEventHandling.

@SuppressWarnings("deprecation")
@Test
public void testMRNewTimelineServiceEventHandling() throws Exception {
    LOG.info("testMRNewTimelineServiceEventHandling start.");
    String testDir = new File("target", getClass().getSimpleName() + "-test_dir").getAbsolutePath();
    String storageDir = testDir + File.separator + "timeline_service_data";
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    // enable new timeline service
    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
    conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class);
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
    // set the file system root directory
    conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT, storageDir);
    // enable aux-service based timeline collectors
    conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
    conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
    conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
    MiniMRYarnCluster cluster = null;
    try {
        cluster = new MiniMRYarnCluster(TestMRTimelineEventHandling.class.getSimpleName(), 1, true);
        cluster.init(conf);
        cluster.start();
        LOG.info("A MiniMRYarnCluster get start.");
        Path inDir = new Path(testDir, "input");
        Path outDir = new Path(testDir, "output");
        LOG.info("Run 1st job which should be successful.");
        JobConf successConf = new JobConf(conf);
        successConf.set("dummy_conf1", UtilsForTests.createConfigValue(51 * 1024));
        successConf.set("dummy_conf2", UtilsForTests.createConfigValue(51 * 1024));
        successConf.set("huge_dummy_conf1", UtilsForTests.createConfigValue(101 * 1024));
        successConf.set("huge_dummy_conf2", UtilsForTests.createConfigValue(101 * 1024));
        RunningJob job = UtilsForTests.runJobSucceed(successConf, inDir, outDir);
        Assert.assertEquals(JobStatus.SUCCEEDED, job.getJobStatus().getState().getValue());
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(new Configuration(cluster.getConfig()));
        yarnClient.start();
        EnumSet<YarnApplicationState> appStates = EnumSet.allOf(YarnApplicationState.class);
        ApplicationId firstAppId = null;
        List<ApplicationReport> apps = yarnClient.getApplications(appStates);
        Assert.assertEquals(apps.size(), 1);
        ApplicationReport appReport = apps.get(0);
        firstAppId = appReport.getApplicationId();
        UtilsForTests.waitForAppFinished(job, cluster);
        checkNewTimelineEvent(firstAppId, appReport, storageDir);
        LOG.info("Run 2nd job which should be failed.");
        job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir);
        Assert.assertEquals(JobStatus.FAILED, job.getJobStatus().getState().getValue());
        apps = yarnClient.getApplications(appStates);
        Assert.assertEquals(apps.size(), 2);
        appReport = apps.get(0).getApplicationId().equals(firstAppId) ? apps.get(0) : apps.get(1);
        checkNewTimelineEvent(firstAppId, appReport, storageDir);
    } finally {
        if (cluster != null) {
            cluster.stop();
        }
        // Cleanup test file
        File testDirFolder = new File(testDir);
        if (testDirFolder.isDirectory()) {
            FileUtils.deleteDirectory(testDirFolder);
        }
    }
}

Example 7 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class TestYarnCLI method testGetApplications.

@Test
public void testGetApplications() throws Exception {
    ApplicationCLI cli = createAndGetAppCLI();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationReport newApplicationReport = ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", "");
    List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
    applicationReports.add(newApplicationReport);
    ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
    ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED, "", "");
    applicationReports.add(newApplicationReport2);
    ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
    ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED, "", "");
    applicationReports.add(newApplicationReport3);
    ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
    ApplicationReport newApplicationReport4 = ApplicationReport.newInstance(applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false, Priority.UNDEFINED, "", "");
    applicationReports.add(newApplicationReport4);
    ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
    ApplicationReport newApplicationReport5 = ApplicationReport.newInstance(applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5), "user5", "queue5", "appname5", "host5", 128, null, YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null, Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", "");
    applicationReports.add(newApplicationReport5);
    ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
    ApplicationReport newApplicationReport6 = ApplicationReport.newInstance(applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6), "user6", "queue6", "appname6", "host6", 129, null, YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", null, new HashSet<String>(), false, Priority.UNDEFINED, "", "");
    applicationReports.add(newApplicationReport6);
    // Test command yarn application -list
    // if the set appStates is empty, RUNNING state will be automatically added
    // to the appStates list
    // the output of yarn application -list should be the same as
    // equals to yarn application -list --appStates RUNNING,ACCEPTED,SUBMITTED
    Set<String> appType1 = new HashSet<String>();
    EnumSet<YarnApplicationState> appState1 = EnumSet.noneOf(YarnApplicationState.class);
    appState1.add(YarnApplicationState.RUNNING);
    appState1.add(YarnApplicationState.ACCEPTED);
    appState1.add(YarnApplicationState.SUBMITTED);
    Set<String> appTag = new HashSet<String>();
    when(client.getApplications(appType1, appState1, appTag)).thenReturn(getApplicationReports(applicationReports, appType1, appState1, appTag, false));
    int result = cli.run(new String[] { "application", "-list" });
    assertEquals(0, result);
    verify(client).getApplications(appType1, appState1, appTag);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType1 + ", states: " + appState1 + " and tags: " + appTag + ")" + ":" + 4);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0005\t             ");
    pw.print("appname\t                YARN\t      user\t     ");
    pw.print("queue\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         53.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0007\t            ");
    pw.print("appname3\t           MAPREDUCE\t     user3\t    ");
    pw.print("queue3\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         73.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0009\t            ");
    pw.print("appname5\t                HIVE\t     user5\t    ");
    pw.print("queue5\t          ACCEPTED\t            ");
    pw.print("KILLED\t         93.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0010\t            ");
    pw.print("appname6\t                 PIG\t     user6\t    ");
    pw.print("queue6\t         SUBMITTED\t            ");
    pw.print("KILLED\t         99.79%");
    pw.println("\t                                N/A");
    pw.close();
    String appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt());
    //Test command yarn application -list --appTypes apptype1,apptype2
    //the output should be the same as
    // yarn application -list --appTypes apptyp1, apptype2 --appStates
    // RUNNING,ACCEPTED,SUBMITTED
    sysOutStream.reset();
    Set<String> appType2 = new HashSet<String>();
    appType2.add("YARN");
    appType2.add("NON-YARN");
    EnumSet<YarnApplicationState> appState2 = EnumSet.noneOf(YarnApplicationState.class);
    appState2.add(YarnApplicationState.RUNNING);
    appState2.add(YarnApplicationState.ACCEPTED);
    appState2.add(YarnApplicationState.SUBMITTED);
    when(client.getApplications(appType2, appState2, appTag)).thenReturn(getApplicationReports(applicationReports, appType2, appState2, appTag, false));
    result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN, ,,  NON-YARN", "   ,, ,," });
    assertEquals(0, result);
    verify(client).getApplications(appType2, appState2, appTag);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType2 + ", states: " + appState2 + " and tags: " + appTag + ")" + ":" + 1);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0005\t             ");
    pw.print("appname\t                YARN\t      user\t     ");
    pw.print("queue\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         53.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt());
    //Test command yarn application -list --appStates appState1,appState2
    sysOutStream.reset();
    Set<String> appType3 = new HashSet<String>();
    EnumSet<YarnApplicationState> appState3 = EnumSet.noneOf(YarnApplicationState.class);
    appState3.add(YarnApplicationState.FINISHED);
    appState3.add(YarnApplicationState.FAILED);
    when(client.getApplications(appType3, appState3, appTag)).thenReturn(getApplicationReports(applicationReports, appType3, appState3, appTag, false));
    result = cli.run(new String[] { "application", "-list", "--appStates", "FINISHED ,, , FAILED", ",,FINISHED" });
    assertEquals(0, result);
    verify(client).getApplications(appType3, appState3, appTag);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType3 + ", states: " + appState3 + " and tags: " + appTag + ")" + ":" + 2);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0006\t            ");
    pw.print("appname2\t            NON-YARN\t     user2\t    ");
    pw.print("queue2\t          FINISHED\t         ");
    pw.print("SUCCEEDED\t         63.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0008\t            ");
    pw.print("appname4\t       NON-MAPREDUCE\t     user4\t    ");
    pw.print("queue4\t            FAILED\t         ");
    pw.print("SUCCEEDED\t         83.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
    // Test command yarn application -list --appTypes apptype1,apptype2
    // --appStates appstate1,appstate2
    sysOutStream.reset();
    Set<String> appType4 = new HashSet<String>();
    appType4.add("YARN");
    appType4.add("NON-YARN");
    EnumSet<YarnApplicationState> appState4 = EnumSet.noneOf(YarnApplicationState.class);
    appState4.add(YarnApplicationState.FINISHED);
    appState4.add(YarnApplicationState.FAILED);
    when(client.getApplications(appType4, appState4, appTag)).thenReturn(getApplicationReports(applicationReports, appType4, appState4, appTag, false));
    result = cli.run(new String[] { "application", "-list", "--appTypes", "YARN,NON-YARN", "--appStates", "FINISHED ,, , FAILED" });
    assertEquals(0, result);
    verify(client).getApplications(appType2, appState2, appTag);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType4 + ", states: " + appState4 + " and tags: " + appTag + ")" + ":" + 1);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0006\t            ");
    pw.print("appname2\t            NON-YARN\t     user2\t    ");
    pw.print("queue2\t          FINISHED\t         ");
    pw.print("SUCCEEDED\t         63.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
    //Test command yarn application -list --appStates with invalid appStates
    sysOutStream.reset();
    result = cli.run(new String[] { "application", "-list", "--appStates", "FINISHED ,, , INVALID" });
    assertEquals(-1, result);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("The application state  INVALID is invalid.");
    pw.print("The valid application state can be one of the following: ");
    StringBuilder sb = new StringBuilder();
    sb.append("ALL,");
    for (YarnApplicationState state : YarnApplicationState.values()) {
        sb.append(state + ",");
    }
    String output = sb.toString();
    pw.println(output.substring(0, output.length() - 1));
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
    //Test command yarn application -list --appStates all
    sysOutStream.reset();
    Set<String> appType5 = new HashSet<String>();
    EnumSet<YarnApplicationState> appState5 = EnumSet.noneOf(YarnApplicationState.class);
    appState5.add(YarnApplicationState.FINISHED);
    when(client.getApplications(appType5, appState5, appTag)).thenReturn(getApplicationReports(applicationReports, appType5, appState5, appTag, true));
    result = cli.run(new String[] { "application", "-list", "--appStates", "FINISHED ,, , ALL" });
    assertEquals(0, result);
    verify(client).getApplications(appType5, appState5, appTag);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType5 + ", states: " + appState5 + " and tags: " + appTag + ")" + ":" + 6);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0005\t             ");
    pw.print("appname\t                YARN\t      user\t     ");
    pw.print("queue\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         53.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0006\t            ");
    pw.print("appname2\t            NON-YARN\t     user2\t    ");
    pw.print("queue2\t          FINISHED\t         ");
    pw.print("SUCCEEDED\t         63.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0007\t            ");
    pw.print("appname3\t           MAPREDUCE\t     user3\t    ");
    pw.print("queue3\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         73.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0008\t            ");
    pw.print("appname4\t       NON-MAPREDUCE\t     user4\t    ");
    pw.print("queue4\t            FAILED\t         ");
    pw.print("SUCCEEDED\t         83.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0009\t            ");
    pw.print("appname5\t                HIVE\t     user5\t    ");
    pw.print("queue5\t          ACCEPTED\t            ");
    pw.print("KILLED\t         93.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0010\t            ");
    pw.print("appname6\t                 PIG\t     user6\t    ");
    pw.print("queue6\t         SUBMITTED\t            ");
    pw.print("KILLED\t         99.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
    // Test command yarn application user case insensitive
    sysOutStream.reset();
    Set<String> appType6 = new HashSet<String>();
    appType6.add("YARN");
    appType6.add("NON-YARN");
    EnumSet<YarnApplicationState> appState6 = EnumSet.noneOf(YarnApplicationState.class);
    appState6.add(YarnApplicationState.FINISHED);
    when(client.getApplications(appType6, appState6, appTag)).thenReturn(getApplicationReports(applicationReports, appType6, appState6, appTag, false));
    result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN, ,,  NON-YARN", "--appStates", "finished" });
    assertEquals(0, result);
    verify(client).getApplications(appType6, appState6, appTag);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType6 + ", states: " + appState6 + " and tags: " + appTag + ")" + ":" + 1);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0006\t            ");
    pw.print("appname2\t            NON-YARN\t     user2\t    ");
    pw.print("queue2\t          FINISHED\t         ");
    pw.print("SUCCEEDED\t         63.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
    // Test command yarn application with tags.
    sysOutStream.reset();
    Set<String> appTag1 = Sets.newHashSet("tag1");
    when(client.getApplications(appType1, appState1, appTag1)).thenReturn(getApplicationReports(applicationReports, appType1, appState1, appTag1, false));
    result = cli.run(new String[] { "application", "-list", "-appTags", "tag1" });
    assertEquals(0, result);
    verify(client).getApplications(appType1, appState1, appTag1);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType1 + ", states: " + appState1 + " and tags: " + appTag1 + ")" + ":" + 2);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0005\t             ");
    pw.print("appname\t                YARN\t      user\t     ");
    pw.print("queue\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         53.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0007\t            ");
    pw.print("appname3\t           MAPREDUCE\t     user3\t    ");
    pw.print("queue3\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         73.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
    sysOutStream.reset();
    EnumSet<YarnApplicationState> appState7 = EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.FAILED);
    when(client.getApplications(appType1, appState7, appTag1)).thenReturn(getApplicationReports(applicationReports, appType1, appState7, appTag1, false));
    result = cli.run(new String[] { "application", "-list", "-appStates", "RUNNING,FAILED", "-appTags", "tag1" });
    assertEquals(0, result);
    verify(client).getApplications(appType1, appState7, appTag1);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType1 + ", states: " + appState7 + " and tags: " + appTag1 + ")" + ":" + 3);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0005\t             ");
    pw.print("appname\t                YARN\t      user\t     ");
    pw.print("queue\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         53.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0007\t            ");
    pw.print("appname3\t           MAPREDUCE\t     user3\t    ");
    pw.print("queue3\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         73.79%");
    pw.println("\t                                N/A");
    pw.print("         application_1234_0008\t            ");
    pw.print("appname4\t       NON-MAPREDUCE\t     user4\t    ");
    pw.print("queue4\t            FAILED\t         ");
    pw.print("SUCCEEDED\t         83.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
    sysOutStream.reset();
    Set<String> appType9 = Sets.newHashSet("YARN");
    Set<String> appTag2 = Sets.newHashSet("tag3");
    when(client.getApplications(appType9, appState1, appTag2)).thenReturn(getApplicationReports(applicationReports, appType9, appState1, appTag2, false));
    result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN", "-appTags", "tag3" });
    assertEquals(0, result);
    verify(client).getApplications(appType9, appState1, appTag2);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType9 + ", states: " + appState1 + " and tags: " + appTag2 + ")" + ":" + 1);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0005\t             ");
    pw.print("appname\t                YARN\t      user\t     ");
    pw.print("queue\t           RUNNING\t         ");
    pw.print("SUCCEEDED\t         53.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
    sysOutStream.reset();
    Set<String> appType10 = Sets.newHashSet("HIVE");
    Set<String> appTag3 = Sets.newHashSet("tag4");
    EnumSet<YarnApplicationState> appState10 = EnumSet.of(YarnApplicationState.ACCEPTED);
    when(client.getApplications(appType10, appState10, appTag3)).thenReturn(getApplicationReports(applicationReports, appType10, appState10, appTag3, false));
    result = cli.run(new String[] { "application", "-list", "-appTypes", "HIVE", "-appStates", "ACCEPTED", "-appTags", "tag4" });
    assertEquals(0, result);
    verify(client).getApplications(appType10, appState10, appTag3);
    baos = new ByteArrayOutputStream();
    pw = new PrintWriter(baos);
    pw.println("Total number of applications (application-types: " + appType10 + ", states: " + appState10 + " and tags: " + appTag3 + ")" + ":" + 1);
    pw.print("                Application-Id\t    Application-Name");
    pw.print("\t    Application-Type");
    pw.print("\t      User\t     Queue\t             State\t       ");
    pw.print("Final-State\t       Progress");
    pw.println("\t                       Tracking-URL");
    pw.print("         application_1234_0009\t            ");
    pw.print("appname5\t                HIVE\t     user5\t    ");
    pw.print("queue5\t          ACCEPTED\t            ");
    pw.print("KILLED\t         93.79%");
    pw.println("\t                                N/A");
    pw.close();
    appsReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appsReportStr, sysOutStream.toString());
    verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt());
}

Example 8 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class TestYarnClient method testGetApplications.

@Test(timeout = 10000)
public void testGetApplications() throws YarnException, IOException {
    Configuration conf = new Configuration();
    final YarnClient client = new MockYarnClient();
    client.init(conf);
    client.start();
    List<ApplicationReport> expectedReports = ((MockYarnClient) client).getReports();
    List<ApplicationReport> reports = client.getApplications();
    Assert.assertEquals(reports, expectedReports);
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("YARN");
    appTypes.add("NON-YARN");
    reports = client.getApplications(appTypes, null);
    Assert.assertEquals(reports.size(), 2);
    Assert.assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports.get(1).getApplicationType().equals("NON-YARN")) || (reports.get(1).getApplicationType().equals("YARN") && reports.get(0).getApplicationType().equals("NON-YARN")));
    for (ApplicationReport report : reports) {
        Assert.assertTrue(expectedReports.contains(report));
    }
    EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);
    appStates.add(YarnApplicationState.FINISHED);
    appStates.add(YarnApplicationState.FAILED);
    reports = client.getApplications(null, appStates);
    Assert.assertEquals(reports.size(), 2);
    Assert.assertTrue((reports.get(0).getApplicationType().equals("NON-YARN") && reports.get(1).getApplicationType().equals("NON-MAPREDUCE")) || (reports.get(1).getApplicationType().equals("NON-YARN") && reports.get(0).getApplicationType().equals("NON-MAPREDUCE")));
    for (ApplicationReport report : reports) {
        Assert.assertTrue(expectedReports.contains(report));
    }
    reports = client.getApplications(appTypes, appStates);
    Assert.assertEquals(reports.size(), 1);
    Assert.assertTrue((reports.get(0).getApplicationType().equals("NON-YARN")));
    for (ApplicationReport report : reports) {
        Assert.assertTrue(expectedReports.contains(report));
    }
    client.stop();
}

Example 9 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class Client method monitorApplication.

/**
   * Monitor the submitted application for completion. 
   * Kill application if time expires. 
   * @param appId Application Id of application to be monitored
   * @return true if application completed successfully
   * @throws YarnException
   * @throws IOException
   */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
    while (true) {
        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);
        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }
}

Example 10 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class MockAsm method newApplication.

public static RMApp newApplication(int i) {
    final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(newAppID(i), 0);
    final Container masterContainer = Records.newRecord(Container.class);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 0);
    masterContainer.setId(containerId);
    masterContainer.setNodeHttpAddress("node:port");
    final String user = newUserName();
    final String name = newAppName();
    final String queue = newQueue();
    final long start = 123456 + i * 1000;
    final long finish = 234567 + i * 1000;
    final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
    YarnApplicationState[] allStates = YarnApplicationState.values();
    final YarnApplicationState state = allStates[i % allStates.length];
    final int maxAppAttempts = i % 1000;
    return new ApplicationBase() {

        @Override
        public ApplicationId getApplicationId() {
            return appAttemptId.getApplicationId();
        }

        @Override
        public String getUser() {
            return user;
        }

        @Override
        public String getName() {
            return name;
        }

        @Override
        public String getApplicationType() {
            return type;
        }

        @Override
        public String getQueue() {
            return queue;
        }

        @Override
        public long getStartTime() {
            return start;
        }

        @Override
        public long getFinishTime() {
            return finish;
        }

        @Override
        public String getTrackingUrl() {
            return null;
        }

        @Override
        public YarnApplicationState createApplicationState() {
            return state;
        }

        @Override
        public StringBuilder getDiagnostics() {
            return new StringBuilder();
        }

        @Override
        public float getProgress() {
            return (float) Math.random();
        }

        @Override
        public FinalApplicationStatus getFinalApplicationStatus() {
            return FinalApplicationStatus.UNDEFINED;
        }

        @Override
        public RMAppAttempt getCurrentAppAttempt() {
            return null;
        }

        @Override
        public int getMaxAppAttempts() {
            return maxAppAttempts;
        }

        @Override
        public Set<String> getApplicationTags() {
            return null;
        }

        @Override
        public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
            ApplicationResourceUsageReport usageReport = ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, 0, 0, 0, 0, 0, 0);
            ApplicationReport report = ApplicationReport.newInstance(getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), getTrackingUrl(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport, null, getProgress(), type, null);
            return report;
        }
    };
}