Examples with ApplicationAttemptId org.apache.hadoop.yarn.api.records.ApplicationAttemptId used on opensource projects

Example 91 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestYarnCLI method testGetContainerReport.

@Test
public void testGetContainerReport() throws Exception {
    ApplicationCLI cli = createAndGetAppCLI();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
    ContainerReport container = ContainerReport.newInstance(containerId, null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE, "http://" + NodeId.newInstance("host", 2345).toString());
    when(client.getContainerReport(any(ContainerId.class))).thenReturn(container);
    int result = cli.run(new String[] { "container", "-status", containerId.toString() });
    assertEquals(0, result);
    verify(client).getContainerReport(containerId);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Container Report : ");
    pw.println("\tContainer-Id : container_1234_0005_01_000001");
    pw.println("\tStart-Time : 1234");
    pw.println("\tFinish-Time : 5678");
    pw.println("\tState : COMPLETE");
    pw.println("\tLOG-URL : logURL");
    pw.println("\tHost : host:1234");
    pw.println("\tNodeHttpAddress : http://host:2345");
    pw.println("\tDiagnostics : diagnosticInfo");
    pw.close();
    String appReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appReportStr, sysOutStream.toString());
    verify(sysOut, times(1)).println(isA(String.class));
}

Example 92 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestYarnCLI method testGetApplicationAttempts.

@Test
public void testGetApplicationAttempts() throws Exception {
    ApplicationCLI cli = createAndGetAppCLI();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance(applicationId, 2);
    ApplicationAttemptReport attemptReport = ApplicationAttemptReport.newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId(attemptId, 1));
    ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport.newInstance(attemptId1, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId(attemptId1, 1));
    List<ApplicationAttemptReport> reports = new ArrayList<ApplicationAttemptReport>();
    reports.add(attemptReport);
    reports.add(attemptReport1);
    when(client.getApplicationAttempts(any(ApplicationId.class))).thenReturn(reports);
    int result = cli.run(new String[] { "applicationattempt", "-list", applicationId.toString() });
    assertEquals(0, result);
    verify(client).getApplicationAttempts(applicationId);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Total number of application attempts :2");
    pw.print("         ApplicationAttempt-Id");
    pw.print("\t               State");
    pw.print("\t                    AM-Container-Id");
    pw.println("\t                       Tracking-URL");
    pw.print("   appattempt_1234_0005_000001");
    pw.print("\t            FINISHED");
    pw.print("\t      container_1234_0005_01_000001");
    pw.println("\t                                url");
    pw.print("   appattempt_1234_0005_000002");
    pw.print("\t            FINISHED");
    pw.print("\t      container_1234_0005_02_000001");
    pw.println("\t                                url");
    pw.close();
    String appReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appReportStr, sysOutStream.toString());
}

Example 93 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestLogsCLI method testFetchFinishedApplictionLogs.

@Test(timeout = 15000)
public void testFetchFinishedApplictionLogs() throws Exception {
    String remoteLogRootDir = "target/logs/";
    Configuration configuration = new Configuration();
    configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
    configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
    FileSystem fs = FileSystem.get(configuration);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
    ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
    final NodeId nodeId = NodeId.newInstance("localhost", 1234);
    // create local logs
    String rootLogDir = "target/LocalLogs";
    Path rootLogDirPath = new Path(rootLogDir);
    if (fs.exists(rootLogDirPath)) {
        fs.delete(rootLogDirPath, true);
    }
    assertTrue(fs.mkdirs(rootLogDirPath));
    Path appLogsDir = new Path(rootLogDirPath, appId.toString());
    if (fs.exists(appLogsDir)) {
        fs.delete(appLogsDir, true);
    }
    assertTrue(fs.mkdirs(appLogsDir));
    List<String> rootLogDirs = Arrays.asList(rootLogDir);
    List<String> logTypes = new ArrayList<String>();
    logTypes.add("syslog");
    // create container logs in localLogDir
    createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes);
    createContainerLogInLocalDir(appLogsDir, containerId2, fs, logTypes);
    // create two logs for container3 in localLogDir
    logTypes.add("stdout");
    logTypes.add("stdout1234");
    createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes);
    Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/logs/application_0_0001");
    if (fs.exists(path)) {
        fs.delete(path, true);
    }
    assertTrue(fs.mkdirs(path));
    // upload container logs into remote directory
    // the first two logs is empty. When we try to read first two logs,
    // we will meet EOF exception, but it will not impact other logs.
    // Other logs should be read successfully.
    uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0, path, fs);
    uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs);
    uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs);
    uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2, path, fs);
    uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId3, path, fs);
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, ugi.getShortUserName());
    LogsCLI cli = new LogsCLIForTest(mockYarnClient) {

        @Override
        public ContainerReport getContainerReport(String containerIdStr) throws YarnException, IOException {
            ContainerReport mockReport = mock(ContainerReport.class);
            doReturn(nodeId).when(mockReport).getAssignedNode();
            doReturn("http://localhost:2345").when(mockReport).getNodeHttpAddress();
            return mockReport;
        }
    };
    cli.setConf(configuration);
    int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files_pattern", ".*" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "*" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    int fullSize = sysOutStream.toByteArray().length;
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "stdout" });
    assertTrue(exitCode == 0);
    assertFalse(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files_pattern", "std*" });
    assertTrue(exitCode == 0);
    assertFalse(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "123" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Can not find any log file matching the pattern: [123] " + "for the application: " + appId.toString()));
    sysErrStream.reset();
    // specify the bytes which is larger than the actual file size,
    // we would get the full logs
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "*", "-size", "10000" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toByteArray().length == fullSize);
    sysOutStream.reset();
    // uploaded two logs for container1. The first log is empty.
    // The second one is not empty.
    // We can still successfully read logs for container1.
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId1.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains("Log Upload Time"));
    assertTrue(!sysOutStream.toString().contains("Logs for container " + containerId1.toString() + " are not present in this log-file."));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "123" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Can not find any log file matching the pattern: [123] " + "for the container: " + containerId3 + " within the application: " + appId.toString()));
    sysErrStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout" });
    assertTrue(exitCode == 0);
    int fullContextSize = sysOutStream.toByteArray().length;
    String fullContext = sysOutStream.toString();
    sysOutStream.reset();
    String logMessage = logMessage(containerId3, "stdout");
    int fileContentSize = logMessage.getBytes().length;
    int tailContentSize = "\nEnd of LogType:stdout\n\n".getBytes().length;
    // specify how many bytes we should get from logs
    // specify a position number, it would get the first n bytes from
    // container log
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", "5" });
    assertTrue(exitCode == 0);
    Assert.assertEquals(new String(logMessage.getBytes(), 0, 5), new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5));
    sysOutStream.reset();
    // specify a negative number, it would get the last n bytes from
    // container log
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", "-5" });
    assertTrue(exitCode == 0);
    Assert.assertEquals(new String(logMessage.getBytes(), logMessage.getBytes().length - 5, 5), new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5));
    sysOutStream.reset();
    long negative = (fullContextSize + 1000) * (-1);
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", Long.toString(negative) });
    assertTrue(exitCode == 0);
    Assert.assertEquals(fullContext, sysOutStream.toString());
    sysOutStream.reset();
    // Uploaded the empty log for container0.
    // We should see the message showing the log for container0
    // are not present.
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId0.toString() });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Logs for container " + containerId0.toString() + " are not present in this log-file."));
    sysErrStream.reset();
    // uploaded two logs for container3. The first log is named as syslog.
    // The second one is named as stdout.
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    sysOutStream.reset();
    // set -log_files option as stdout
    // should only print log with the name as stdout
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(!sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    sysOutStream.reset();
    YarnClient mockYarnClientWithException = createMockYarnClientWithException();
    cli = new LogsCLIForTest(mockYarnClientWithException);
    cli.setConf(configuration);
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
    sysOutStream.reset();
    // The same should also work without the applicationId
    exitCode = cli.run(new String[] { "-containerId", containerId3.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-containerId", "invalid_container" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Invalid ContainerId specified"));
    sysErrStream.reset();
    fs.delete(new Path(remoteLogRootDir), true);
    fs.delete(new Path(rootLogDir), true);
}

Example 94 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestLogsCLI method testSaveContainerLogsLocally.

@Test(timeout = 15000)
public void testSaveContainerLogsLocally() throws Exception {
    String remoteLogRootDir = "target/logs/";
    String rootLogDir = "target/LocalLogs";
    String localDir = "target/SaveLogs";
    Path localPath = new Path(localDir);
    Configuration configuration = new Configuration();
    configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
    configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
    FileSystem fs = FileSystem.get(configuration);
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
    containerIds.add(containerId1);
    containerIds.add(containerId2);
    List<NodeId> nodeIds = new ArrayList<NodeId>();
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    NodeId nodeId2 = NodeId.newInstance("test", 4567);
    nodeIds.add(nodeId);
    nodeIds.add(nodeId2);
    try {
        createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, appId, containerIds, nodeIds);
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI cli = new LogsCLIForTest(mockYarnClient);
        cli.setConf(configuration);
        int exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-out", localPath.toString() });
        assertTrue(exitCode == 0);
        // make sure we created a dir named as node id
        FileStatus[] nodeDir = fs.listStatus(localPath);
        Arrays.sort(nodeDir);
        assertTrue(nodeDir.length == 2);
        assertTrue(nodeDir[0].getPath().getName().contains(LogAggregationUtils.getNodeString(nodeId)));
        assertTrue(nodeDir[1].getPath().getName().contains(LogAggregationUtils.getNodeString(nodeId2)));
        FileStatus[] container1Dir = fs.listStatus(nodeDir[0].getPath());
        assertTrue(container1Dir.length == 1);
        assertTrue(container1Dir[0].getPath().getName().equals(containerId1.toString()));
        String container1 = readContainerContent(container1Dir[0].getPath(), fs);
        assertTrue(container1.contains(logMessage(containerId1, "syslog")));
        FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath());
        assertTrue(container2Dir.length == 1);
        assertTrue(container2Dir[0].getPath().getName().equals(containerId2.toString()));
        String container2 = readContainerContent(container2Dir[0].getPath(), fs);
        assertTrue(container2.contains(logMessage(containerId2, "syslog")));
    } finally {
        fs.delete(new Path(remoteLogRootDir), true);
        fs.delete(new Path(rootLogDir), true);
        fs.delete(localPath, true);
    }
}

Example 95 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestRMAppAttemptTransitions method testContainersCleanupForLastAttempt.

@SuppressWarnings("deprecation")
@Test
public void testContainersCleanupForLastAttempt() {
    // create a failed attempt.
    applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    when(submissionContext.getKeepContainersAcrossApplicationAttempts()).thenReturn(true);
    when(submissionContext.getMaxAppAttempts()).thenReturn(1);
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    ContainerStatus cs1 = ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "some error", 123);
    ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs1, anyNodeId));
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState());
    assertFalse(transferStateFromPreviousAttempt);
    verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}