Examples with YarnClient org.apache.hadoop.yarn.client.api.YarnClient used on opensource projects

Example 81 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testInvalidApplicationId.

@Test(timeout = 5000l)
public void testInvalidApplicationId() throws Exception {
    Configuration conf = new YarnConfiguration();
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName());
    LogsCLI cli = new LogsCLIForTest(mockYarnClient);
    cli.setConf(conf);
    int exitCode = cli.run(new String[] { "-applicationId", "not_an_app_id" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified"));
}

Example 82 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method createMockYarnClientWithException.

private YarnClient createMockYarnClientWithException(YarnApplicationState appState, String user) throws YarnException, IOException {
    YarnClient mockClient = mock(YarnClient.class);
    ApplicationReport mockAppReport = mock(ApplicationReport.class);
    doReturn(user).when(mockAppReport).getUser();
    doReturn(appState).when(mockAppReport).getYarnApplicationState();
    doReturn(mockAppReport).when(mockClient).getApplicationReport(any(ApplicationId.class));
    doThrow(new YarnException()).when(mockClient).getContainerReport(any(ContainerId.class));
    return mockClient;
}

Example 83 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method createMockYarnClientWithException.

private YarnClient createMockYarnClientWithException() throws YarnException, IOException {
    YarnClient mockClient = mock(YarnClient.class);
    doThrow(new YarnException()).when(mockClient).getApplicationReport(any(ApplicationId.class));
    doThrow(new YarnException()).when(mockClient).getContainerReport(any(ContainerId.class));
    return mockClient;
}

Example 84 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testFetchApplictionLogsAsAnotherUser.

@Test(timeout = 15000)
public void testFetchApplictionLogsAsAnotherUser() throws Exception {
    String remoteLogRootDir = "target/logs/";
    String rootLogDir = "target/LocalLogs";
    String testUser = "test";
    UserGroupInformation testUgi = UserGroupInformation.createRemoteUser(testUser);
    Configuration configuration = new Configuration();
    configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
    configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
    FileSystem fs = FileSystem.get(configuration);
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    try {
        Path rootLogDirPath = new Path(rootLogDir);
        if (fs.exists(rootLogDirPath)) {
            fs.delete(rootLogDirPath, true);
        }
        assertTrue(fs.mkdirs(rootLogDirPath));
        // create local app dir for app
        final Path appLogsDir = new Path(rootLogDirPath, appId.toString());
        if (fs.exists(appLogsDir)) {
            fs.delete(appLogsDir, true);
        }
        assertTrue(fs.mkdirs(appLogsDir));
        List<String> rootLogDirs = Arrays.asList(rootLogDir);
        List<String> logTypes = new ArrayList<String>();
        logTypes.add("syslog");
        // create container logs in localLogDir for app
        createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
        // create the remote app dir for app
        // but for a different user testUser"
        Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
        if (fs.exists(path)) {
            fs.delete(path, true);
        }
        assertTrue(fs.mkdirs(path));
        // upload container logs for app into remote dir
        uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs, nodeId, containerId, path, fs);
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, testUgi.getShortUserName());
        LogsCLI cli = new LogsCLIForTest(mockYarnClient);
        cli.setConf(configuration);
        // Verify that we can get the application logs by specifying
        // a correct appOwner
        int exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-appOwner", testUser });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId, "syslog")));
        sysOutStream.reset();
        // Verify that we can not get the application logs
        // if an invalid user is specified
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-appOwner", "invalid" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Can not find the logs " + "for the application: " + appId.toString()));
        sysErrStream.reset();
        // Verify that we do not specify appOwner, and can not
        // get appReport from RM, we still can figure out the appOwner
        // and can get app logs successfully.
        YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
        cli = new LogsCLIForTest(mockYarnClient2);
        cli.setConf(configuration);
        exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId, "syslog")));
        sysOutStream.reset();
        // Verify that we could get the err message "Can not find the appOwner"
        // if we do not specify the appOwner, can not get appReport, and
        // the app does not exist in remote dir.
        ApplicationId appId2 = ApplicationId.newInstance(System.currentTimeMillis(), 2);
        exitCode = cli.run(new String[] { "-applicationId", appId2.toString() });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Can not find the appOwner"));
        sysErrStream.reset();
        // Verify that we could not get appOwner
        // because we don't have file-system permissions
        ApplicationId appTest = ApplicationId.newInstance(System.currentTimeMillis(), 1000);
        String priorityUser = "priority";
        Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser + "/logs/" + appTest);
        if (fs.exists(pathWithoutPerm)) {
            fs.delete(pathWithoutPerm, true);
        }
        // The user will not have read permission for this directory.
        // To mimic the scenario that the user can not get file status
        FsPermission permission = FsPermission.createImmutable((short) 01300);
        assertTrue(fs.mkdirs(pathWithoutPerm, permission));
        exitCode = cli.run(new String[] { "-applicationId", appTest.toString() });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Can not find the logs for the application: " + appTest.toString()));
        sysErrStream.reset();
    } finally {
        fs.delete(new Path(remoteLogRootDir), true);
        fs.delete(new Path(rootLogDir), true);
    }
}

Example 85 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testGetRunningContainerLogs.

@Test(timeout = 5000)
public void testGetRunningContainerLogs() throws Exception {
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    // Create a mock ApplicationAttempt Report
    ApplicationAttemptReport mockAttemptReport = mock(ApplicationAttemptReport.class);
    doReturn(appAttemptId).when(mockAttemptReport).getApplicationAttemptId();
    List<ApplicationAttemptReport> attemptReports = Arrays.asList(mockAttemptReport);
    // Create one mock containerReport
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    ContainerReport mockContainerReport1 = mock(ContainerReport.class);
    doReturn(containerId1).when(mockContainerReport1).getContainerId();
    doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
    doReturn("http://localhost:2345").when(mockContainerReport1).getNodeHttpAddress();
    doReturn(ContainerState.RUNNING).when(mockContainerReport1).getContainerState();
    List<ContainerReport> containerReports = Arrays.asList(mockContainerReport1);
    // Mock the YarnClient, and it would report the previous created
    // mockAttemptReport and previous two created mockContainerReports
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.RUNNING, ugi.getShortUserName(), true, attemptReports, containerReports);
    doReturn(mockContainerReport1).when(mockYarnClient).getContainerReport(any(ContainerId.class));
    // create local logs
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(configuration);
    String rootLogDir = "target/LocalLogs";
    Path rootLogDirPath = new Path(rootLogDir);
    if (fs.exists(rootLogDirPath)) {
        fs.delete(rootLogDirPath, true);
    }
    assertTrue(fs.mkdirs(rootLogDirPath));
    Path appLogsDir = new Path(rootLogDirPath, appId.toString());
    if (fs.exists(appLogsDir)) {
        fs.delete(appLogsDir, true);
    }
    assertTrue(fs.mkdirs(appLogsDir));
    String fileName = "syslog";
    List<String> logTypes = new ArrayList<String>();
    logTypes.add(fileName);
    // create container logs in localLogDir
    createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes);
    Path containerDirPath = new Path(appLogsDir, containerId1.toString());
    Path logPath = new Path(containerDirPath, fileName);
    File logFile = new File(logPath.toString());
    final FileInputStream fis = new FileInputStream(logFile);
    try {
        LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
        Set<String> logsSet = new HashSet<String>();
        logsSet.add(fileName);
        doReturn(logsSet).when(cli).getMatchedContainerLogFiles(any(ContainerLogsRequest.class), anyBoolean());
        ClientResponse mockReponse = mock(ClientResponse.class);
        doReturn(Status.OK).when(mockReponse).getStatusInfo();
        doReturn(fis).when(mockReponse).getEntityInputStream();
        doReturn(mockReponse).when(cli).getResponeFromNMWebService(any(Configuration.class), any(Client.class), any(ContainerLogsRequest.class), anyString());
        cli.setConf(new YarnConfiguration());
        int exitCode = cli.run(new String[] { "-containerId", containerId1.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        sysOutStream.reset();
    } finally {
        IOUtils.closeQuietly(fis);
        fs.delete(new Path(rootLogDir), true);
    }
}