Examples with ApplicationAttemptReport org.apache.hadoop.yarn.api.records.ApplicationAttemptReport used on opensource projects

Example 31 with ApplicationAttemptReport

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptReport in project hadoop by apache.

the class UnmanagedAMLauncher method run.

public boolean run() throws IOException, YarnException {
    LOG.info("Starting Client");
    // Connect to ResourceManager
    rmClient.start();
    try {
        // Create launch context for app master
        LOG.info("Setting up application submission context for ASM");
        ApplicationSubmissionContext appContext = rmClient.createApplication().getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();
        // set the application name
        appContext.setApplicationName(appName);
        // Set the priority for the application master
        Priority pri = Records.newRecord(Priority.class);
        pri.setPriority(amPriority);
        appContext.setPriority(pri);
        // Set the queue to which this application is to be submitted in the RM
        appContext.setQueue(amQueue);
        // Set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        appContext.setAMContainerSpec(amContainer);
        // unmanaged AM
        appContext.setUnmanagedAM(true);
        LOG.info("Setting unmanaged AM");
        // Submit the application to the applications manager
        LOG.info("Submitting application to ASM");
        rmClient.submitApplication(appContext);
        ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
            // Monitor the application attempt to wait for launch state
            ApplicationAttemptReport attemptReport = monitorCurrentAppAttempt(appId, YarnApplicationAttemptState.LAUNCHED);
            ApplicationAttemptId attemptId = attemptReport.getApplicationAttemptId();
            LOG.info("Launching AM with application attempt id " + attemptId);
            // launch AM
            launchAM(attemptId);
            // Monitor the application for end state
            appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        }
        YarnApplicationState appState = appReport.getYarnApplicationState();
        FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();
        LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus);
        boolean success;
        if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) {
            LOG.info("Application has completed successfully.");
            success = true;
        } else {
            LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString());
            success = false;
        }
        return success;
    } finally {
        rmClient.stop();
    }
}

Example 32 with ApplicationAttemptReport

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptReport in project hadoop by apache.

the class TestAHSClient method testGetApplicationAttempts.

@Test(timeout = 10000)
public void testGetApplicationAttempts() throws YarnException, IOException {
    Configuration conf = new Configuration();
    final AHSClient client = new MockAHSClient();
    client.init(conf);
    client.start();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    List<ApplicationAttemptReport> reports = client.getApplicationAttempts(applicationId);
    Assert.assertNotNull(reports);
    Assert.assertEquals(reports.get(0).getApplicationAttemptId(), ApplicationAttemptId.newInstance(applicationId, 1));
    Assert.assertEquals(reports.get(1).getApplicationAttemptId(), ApplicationAttemptId.newInstance(applicationId, 2));
    client.stop();
}

Example 33 with ApplicationAttemptReport

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptReport in project hadoop by apache.

the class TestLogsCLI method testFetchRunningApplicationLogs.

@Test(timeout = 5000)
public void testFetchRunningApplicationLogs() throws Exception {
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    // Create a mock ApplicationAttempt Report
    ApplicationAttemptReport mockAttemptReport = mock(ApplicationAttemptReport.class);
    doReturn(appAttemptId).when(mockAttemptReport).getApplicationAttemptId();
    List<ApplicationAttemptReport> attemptReports = Arrays.asList(mockAttemptReport);
    // Create two mock containerReports
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    ContainerReport mockContainerReport1 = mock(ContainerReport.class);
    doReturn(containerId1).when(mockContainerReport1).getContainerId();
    doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
    doReturn("http://localhost:2345").when(mockContainerReport1).getNodeHttpAddress();
    ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
    ContainerReport mockContainerReport2 = mock(ContainerReport.class);
    doReturn(containerId2).when(mockContainerReport2).getContainerId();
    doReturn(nodeId).when(mockContainerReport2).getAssignedNode();
    doReturn("http://localhost:2345").when(mockContainerReport2).getNodeHttpAddress();
    List<ContainerReport> containerReports = Arrays.asList(mockContainerReport1, mockContainerReport2);
    // Mock the YarnClient, and it would report the previous created
    // mockAttemptReport and previous two created mockContainerReports
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.RUNNING, ugi.getShortUserName(), true, attemptReports, containerReports);
    LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
    doReturn(0).when(cli).printContainerLogsFromRunningApplication(any(Configuration.class), any(ContainerLogsRequest.class), any(LogCLIHelpers.class), anyBoolean());
    cli.setConf(new YarnConfiguration());
    int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
    assertTrue(exitCode == 0);
    ArgumentCaptor<ContainerLogsRequest> logsRequestCaptor = ArgumentCaptor.forClass(ContainerLogsRequest.class);
    // we have two container reports, so make sure we have called
    // printContainerLogsFromRunningApplication twice
    verify(cli, times(2)).printContainerLogsFromRunningApplication(any(Configuration.class), logsRequestCaptor.capture(), any(LogCLIHelpers.class), anyBoolean());
    // Verify that the log-type is "ALL"
    List<ContainerLogsRequest> capturedRequests = logsRequestCaptor.getAllValues();
    Assert.assertEquals(2, capturedRequests.size());
    Set<String> logTypes0 = capturedRequests.get(0).getLogTypes();
    Set<String> logTypes1 = capturedRequests.get(1).getLogTypes();
    Assert.assertTrue(logTypes0.contains("ALL") && (logTypes0.size() == 1));
    Assert.assertTrue(logTypes1.contains("ALL") && (logTypes1.size() == 1));
    mockYarnClient = createMockYarnClientWithException(YarnApplicationState.RUNNING, ugi.getShortUserName());
    LogsCLI cli2 = spy(new LogsCLIForTest(mockYarnClient));
    doReturn(0).when(cli2).printContainerLogsFromRunningApplication(any(Configuration.class), any(ContainerLogsRequest.class), any(LogCLIHelpers.class), anyBoolean());
    doReturn("123").when(cli2).getNodeHttpAddressFromRMWebString(any(ContainerLogsRequest.class));
    cli2.setConf(new YarnConfiguration());
    ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100);
    exitCode = cli2.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId100.toString(), "-nodeAddress", "NM:1234" });
    assertTrue(exitCode == 0);
    verify(cli2, times(1)).printContainerLogsFromRunningApplication(any(Configuration.class), logsRequestCaptor.capture(), any(LogCLIHelpers.class), anyBoolean());
}

Example 34 with ApplicationAttemptReport

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptReport in project hadoop by apache.

the class TestLogsCLI method testGetRunningContainerLogs.

@Test(timeout = 5000)
public void testGetRunningContainerLogs() throws Exception {
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    // Create a mock ApplicationAttempt Report
    ApplicationAttemptReport mockAttemptReport = mock(ApplicationAttemptReport.class);
    doReturn(appAttemptId).when(mockAttemptReport).getApplicationAttemptId();
    List<ApplicationAttemptReport> attemptReports = Arrays.asList(mockAttemptReport);
    // Create one mock containerReport
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    ContainerReport mockContainerReport1 = mock(ContainerReport.class);
    doReturn(containerId1).when(mockContainerReport1).getContainerId();
    doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
    doReturn("http://localhost:2345").when(mockContainerReport1).getNodeHttpAddress();
    doReturn(ContainerState.RUNNING).when(mockContainerReport1).getContainerState();
    List<ContainerReport> containerReports = Arrays.asList(mockContainerReport1);
    // Mock the YarnClient, and it would report the previous created
    // mockAttemptReport and previous two created mockContainerReports
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.RUNNING, ugi.getShortUserName(), true, attemptReports, containerReports);
    doReturn(mockContainerReport1).when(mockYarnClient).getContainerReport(any(ContainerId.class));
    // create local logs
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(configuration);
    String rootLogDir = "target/LocalLogs";
    Path rootLogDirPath = new Path(rootLogDir);
    if (fs.exists(rootLogDirPath)) {
        fs.delete(rootLogDirPath, true);
    }
    assertTrue(fs.mkdirs(rootLogDirPath));
    Path appLogsDir = new Path(rootLogDirPath, appId.toString());
    if (fs.exists(appLogsDir)) {
        fs.delete(appLogsDir, true);
    }
    assertTrue(fs.mkdirs(appLogsDir));
    String fileName = "syslog";
    List<String> logTypes = new ArrayList<String>();
    logTypes.add(fileName);
    // create container logs in localLogDir
    createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes);
    Path containerDirPath = new Path(appLogsDir, containerId1.toString());
    Path logPath = new Path(containerDirPath, fileName);
    File logFile = new File(logPath.toString());
    final FileInputStream fis = new FileInputStream(logFile);
    try {
        LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
        Set<String> logsSet = new HashSet<String>();
        logsSet.add(fileName);
        doReturn(logsSet).when(cli).getMatchedContainerLogFiles(any(ContainerLogsRequest.class), anyBoolean());
        ClientResponse mockReponse = mock(ClientResponse.class);
        doReturn(Status.OK).when(mockReponse).getStatusInfo();
        doReturn(fis).when(mockReponse).getEntityInputStream();
        doReturn(mockReponse).when(cli).getResponeFromNMWebService(any(Configuration.class), any(Client.class), any(ContainerLogsRequest.class), anyString());
        cli.setConf(new YarnConfiguration());
        int exitCode = cli.run(new String[] { "-containerId", containerId1.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        sysOutStream.reset();
    } finally {
        IOUtils.closeQuietly(fis);
        fs.delete(new Path(rootLogDir), true);
    }
}

Example 35 with ApplicationAttemptReport

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptReport in project hadoop by apache.

the class TestYarnCLI method testGetApplicationAttemptReport.

@Test
public void testGetApplicationAttemptReport() throws Exception {
    ApplicationCLI cli = createAndGetAppCLI();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ApplicationAttemptReport attemptReport = ApplicationAttemptReport.newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId(attemptId, 1), 1000l, 2000l);
    when(client.getApplicationAttemptReport(any(ApplicationAttemptId.class))).thenReturn(attemptReport);
    int result = cli.run(new String[] { "applicationattempt", "-status", attemptId.toString() });
    assertEquals(0, result);
    verify(client).getApplicationAttemptReport(attemptId);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Application Attempt Report : ");
    pw.println("\tApplicationAttempt-Id : appattempt_1234_0005_000001");
    pw.println("\tState : FINISHED");
    pw.println("\tAMContainer : container_1234_0005_01_000001");
    pw.println("\tTracking-URL : url");
    pw.println("\tRPC Port : 124");
    pw.println("\tAM Host : host");
    pw.println("\tDiagnostics : diagnostics");
    pw.close();
    String appReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appReportStr, sysOutStream.toString());
    verify(sysOut, times(1)).println(isA(String.class));
}