Examples with PrintWriter java.io.PrintWriter used on opensource projects

Example 71 with PrintWriter

use of java.io.PrintWriter in project hadoop by apache.

the class TestContainerLogsPage method testContainerLogPageAccess.

@Test(timeout = 10000)
public void testContainerLogPageAccess() throws IOException {
    // SecureIOUtils require Native IO to be enabled. This test will run
    // only if it is enabled.
    assumeTrue(NativeIO.isAvailable());
    String user = "randomUser" + System.currentTimeMillis();
    File absLogDir = null, appDir = null, containerDir = null, syslog = null;
    try {
        // target log directory
        absLogDir = new File("target", TestContainerLogsPage.class.getSimpleName() + "LogDir").getAbsoluteFile();
        absLogDir.mkdir();
        Configuration conf = new Configuration();
        conf.set(YarnConfiguration.NM_LOG_DIRS, absLogDir.toURI().toString());
        conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
        UserGroupInformation.setConfiguration(conf);
        NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
        healthChecker.init(conf);
        LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
        // Add an application and the corresponding containers
        RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
        long clusterTimeStamp = 1234;
        ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
        Application app = mock(Application.class);
        when(app.getAppId()).thenReturn(appId);
        // Making sure that application returns a random user. This is required
        // for SecureIOUtils' file owner check.
        when(app.getUser()).thenReturn(user);
        ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
        ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
        // Testing secure read access for log files
        // Creating application and container directory and syslog file.
        appDir = new File(absLogDir, appId.toString());
        appDir.mkdir();
        containerDir = new File(appDir, container1.toString());
        containerDir.mkdir();
        syslog = new File(containerDir, "syslog");
        syslog.createNewFile();
        BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(syslog));
        out.write("Log file Content".getBytes());
        out.close();
        Context context = mock(Context.class);
        ConcurrentMap<ApplicationId, Application> appMap = new ConcurrentHashMap<ApplicationId, Application>();
        appMap.put(appId, app);
        when(context.getApplications()).thenReturn(appMap);
        ConcurrentHashMap<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>();
        when(context.getContainers()).thenReturn(containers);
        when(context.getLocalDirsHandler()).thenReturn(dirsHandler);
        MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1);
        container.setState(ContainerState.RUNNING);
        context.getContainers().put(container1, container);
        ContainersLogsBlock cLogsBlock = new ContainersLogsBlock(context);
        Map<String, String> params = new HashMap<String, String>();
        params.put(YarnWebParams.CONTAINER_ID, container1.toString());
        params.put(YarnWebParams.CONTAINER_LOG_TYPE, "syslog");
        Injector injector = WebAppTests.testPage(ContainerLogsPage.class, ContainersLogsBlock.class, cLogsBlock, params, (Module[]) null);
        PrintWriter spyPw = WebAppTests.getPrintWriter(injector);
        verify(spyPw).write("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : syslog");
    } finally {
        if (syslog != null) {
            syslog.delete();
        }
        if (containerDir != null) {
            containerDir.delete();
        }
        if (appDir != null) {
            appDir.delete();
        }
        if (absLogDir != null) {
            absLogDir.delete();
        }
    }
}

Example 72 with PrintWriter

use of java.io.PrintWriter in project hadoop by apache.

the class TestContainersMonitor method testContainerKillOnMemoryOverflow.

@Test
public void testContainerKillOnMemoryOverflow() throws IOException, InterruptedException, YarnException {
    if (!ProcfsBasedProcessTree.isAvailable()) {
        return;
    }
    containerManager.start();
    File scriptFile = new File(tmpDir, "scriptFile.sh");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    File processStartFile = new File(tmpDir, "start_file.txt").getAbsoluteFile();
    // So that start file is readable by the
    fileWriter.write("\numask 0");
    // test.
    fileWriter.write("\necho Hello World! > " + processStartFile);
    fileWriter.write("\necho $$ >> " + processStartFile);
    fileWriter.write("\nsleep 15");
    fileWriter.close();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    // ////// Construct the Container-id
    ApplicationId appId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = new ArrayList<String>();
    commands.add("/bin/bash");
    commands.add(scriptFile.getAbsolutePath());
    containerLaunchContext.setCommands(commands);
    Resource r = BuilderUtils.newResource(0, 0);
    ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER, Priority.newInstance(0), 0);
    Token containerToken = BuilderUtils.newContainerToken(context.getNodeId(), containerManager.getContext().getContainerTokenSecretManager().createPassword(containerIdentifier), containerIdentifier);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    int timeoutSecs = 0;
    while (!processStartFile.exists() && timeoutSecs++ < 20) {
        Thread.sleep(1000);
        LOG.info("Waiting for process start-file to be created");
    }
    Assert.assertTrue("ProcessStartFile doesn't exist!", processStartFile.exists());
    // Now verify the contents of the file
    BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
    Assert.assertEquals("Hello World!", reader.readLine());
    // Get the pid of the process
    String pid = reader.readLine().trim();
    // No more lines
    Assert.assertEquals(null, reader.readLine());
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE, 60);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    Assert.assertEquals(ContainerExitStatus.KILLED_EXCEEDED_VMEM, containerStatus.getExitStatus());
    String expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. " + "Killing container.\nDump of the process-tree for " + cId + " :\n";
    Pattern pat = Pattern.compile(expectedMsgPattern);
    Assert.assertEquals("Expected message pattern is: " + expectedMsgPattern + "\n\nObserved message is: " + containerStatus.getDiagnostics(), true, pat.matcher(containerStatus.getDiagnostics()).find());
    // Assert that the process is not alive anymore
    Assert.assertFalse("Process is still alive!", exec.signalContainer(new ContainerSignalContext.Builder().setUser(user).setPid(pid).setSignal(Signal.NULL).build()));
}

Example 73 with PrintWriter

use of java.io.PrintWriter in project hadoop by apache.

the class TestScriptBasedNodeLabelsProvider method writeNodeLabelsScriptFile.

private void writeNodeLabelsScriptFile(String scriptStr, boolean setExecutable) throws IOException {
    PrintWriter pw = null;
    try {
        FileUtil.setWritable(nodeLabelsScriptFile, true);
        FileUtil.setReadable(nodeLabelsScriptFile, true);
        pw = new PrintWriter(new FileOutputStream(nodeLabelsScriptFile));
        pw.println(scriptStr);
        pw.flush();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail();
    } finally {
        if (null != pw) {
            pw.close();
        }
    }
    FileUtil.setExecutable(nodeLabelsScriptFile, setExecutable);
}

Example 74 with PrintWriter

use of java.io.PrintWriter in project hadoop by apache.

the class TestNMWebServices method testContainerLogs.

private void testContainerLogs(WebResource r, ContainerId containerId) throws IOException {
    final String containerIdStr = containerId.toString();
    final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
    final ApplicationId appId = appAttemptId.getApplicationId();
    final String appIdStr = appId.toString();
    final String filename = "logfile1";
    final String logMessage = "log message\n";
    nmContext.getApplications().put(appId, new ApplicationImpl(null, "user", appId, null, nmContext));
    MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), new Configuration(), "user", appId, 1);
    container.setState(ContainerState.RUNNING);
    nmContext.getContainers().put(containerId, container);
    // write out log file
    Path path = dirsHandler.getLogPathForWrite(ContainerLaunch.getRelativeContainerLogDir(appIdStr, containerIdStr) + "/" + filename, false);
    File logFile = new File(path.toUri().getPath());
    logFile.deleteOnExit();
    assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
    PrintWriter pw = new PrintWriter(logFile);
    pw.print(logMessage);
    pw.close();
    // ask for it
    ClientResponse response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    String responseText = response.getEntity(String.class);
    String responseLogMessage = getLogContext(responseText);
    assertEquals(logMessage, responseLogMessage);
    int fullTextSize = responseLogMessage.getBytes().length;
    // specify how many bytes we should get from logs
    // specify a position number, it would get the first n bytes from
    // container log
    response = r.path(filename).queryParam("size", "5").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(5, responseLogMessage.getBytes().length);
    assertEquals(new String(logMessage.getBytes(), 0, 5), responseLogMessage);
    assertTrue(fullTextSize >= responseLogMessage.getBytes().length);
    // specify the bytes which is larger than the actual file size,
    // we would get the full logs
    response = r.path(filename).queryParam("size", "10000").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(fullTextSize, responseLogMessage.getBytes().length);
    assertEquals(logMessage, responseLogMessage);
    // specify a negative number, it would get the last n bytes from
    // container log
    response = r.path(filename).queryParam("size", "-5").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(5, responseLogMessage.getBytes().length);
    assertEquals(new String(logMessage.getBytes(), logMessage.getBytes().length - 5, 5), responseLogMessage);
    assertTrue(fullTextSize >= responseLogMessage.getBytes().length);
    response = r.path(filename).queryParam("size", "-10000").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals("text/plain; charset=utf-8", response.getType().toString());
    assertEquals(fullTextSize, responseLogMessage.getBytes().length);
    assertEquals(logMessage, responseLogMessage);
    // ask and download it
    response = r.path(filename).queryParam("format", "octet-stream").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(logMessage, responseLogMessage);
    assertEquals(200, response.getStatus());
    assertEquals("application/octet-stream; charset=utf-8", response.getType().toString());
    // specify a invalid format value
    response = r.path(filename).queryParam("format", "123").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    assertEquals("The valid values for the parameter : format are " + WebAppUtils.listSupportedLogContentType(), responseText);
    assertEquals(400, response.getStatus());
    // ask for file that doesn't exist and it will re-direct to
    // the log server
    URI requestURI = r.path("uhhh").getURI();
    String redirectURL = getRedirectURL(requestURI.toString());
    assertTrue(redirectURL != null);
    assertTrue(redirectURL.contains(LOGSERVICEWSADDR));
    // Get container log files' name
    WebResource r1 = resource();
    response = r1.path("ws").path("v1").path("node").path("containers").path(containerIdStr).path("logs").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(200, response.getStatus());
    List<ContainerLogsInfo> responseList = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseList.size() == 1);
    assertEquals(responseList.get(0).getLogType(), ContainerLogAggregationType.LOCAL.toString());
    List<PerContainerLogFileInfo> logMeta = responseList.get(0).getContainerLogsInfo();
    assertTrue(logMeta.size() == 1);
    assertEquals(logMeta.get(0).getFileName(), filename);
    // now create an aggregated log in Remote File system
    File tempLogDir = new File("target", TestNMWebServices.class.getSimpleName() + "temp-log-dir");
    try {
        String aggregatedLogFile = filename + "-aggregated";
        String aggregatedLogMessage = "This is aggregated ;og.";
        TestContainerLogsUtils.createContainerLogFileInRemoteFS(nmContext.getConf(), FileSystem.get(nmContext.getConf()), tempLogDir.getAbsolutePath(), containerId, nmContext.getNodeId(), aggregatedLogFile, "user", aggregatedLogMessage, true);
        r1 = resource();
        response = r1.path("ws").path("v1").path("node").path("containers").path(containerIdStr).path("logs").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(200, response.getStatus());
        responseList = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
        });
        assertEquals(responseList.size(), 2);
        for (ContainerLogsInfo logInfo : responseList) {
            if (logInfo.getLogType().equals(ContainerLogAggregationType.AGGREGATED.toString())) {
                List<PerContainerLogFileInfo> meta = logInfo.getContainerLogsInfo();
                assertTrue(meta.size() == 1);
                assertEquals(meta.get(0).getFileName(), aggregatedLogFile);
            } else {
                assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString());
                List<PerContainerLogFileInfo> meta = logInfo.getContainerLogsInfo();
                assertTrue(meta.size() == 1);
                assertEquals(meta.get(0).getFileName(), filename);
            }
        }
        // Test whether we could get aggregated log as well
        TestContainerLogsUtils.createContainerLogFileInRemoteFS(nmContext.getConf(), FileSystem.get(nmContext.getConf()), tempLogDir.getAbsolutePath(), containerId, nmContext.getNodeId(), filename, "user", aggregatedLogMessage, true);
        response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
        responseText = response.getEntity(String.class);
        assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.AGGREGATED));
        assertTrue(responseText.contains(aggregatedLogMessage));
        assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.LOCAL));
        assertTrue(responseText.contains(logMessage));
    } finally {
        FileUtil.fullyDelete(tempLogDir);
    }
    // After container is completed, it is removed from nmContext
    nmContext.getContainers().remove(containerId);
    Assert.assertNull(nmContext.getContainers().get(containerId));
    response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    assertTrue(responseText.contains(logMessage));
}

Example 75 with PrintWriter

use of java.io.PrintWriter in project hadoop by apache.

the class TestYarnCLI method testGetQueueInfoWithEmptyNodeLabel.

@Test
public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
    QueueCLI cli = createAndGetQueueCLI();
    QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f, null, null, QueueState.RUNNING, null, null, null, true);
    when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
    int result = cli.run(new String[] { "-status", "queueA" });
    assertEquals(0, result);
    verify(client).getQueueInfo("queueA");
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Queue Information : ");
    pw.println("Queue Name : " + "queueA");
    pw.println("\tState : " + "RUNNING");
    pw.println("\tCapacity : " + "40.0%");
    pw.println("\tCurrent Capacity : " + "50.0%");
    pw.println("\tMaximum Capacity : " + "80.0%");
    pw.println("\tDefault Node Label expression : " + NodeLabel.DEFAULT_NODE_LABEL_PARTITION);
    pw.println("\tAccessible Node Labels : ");
    pw.println("\tPreemption : " + "disabled");
    pw.close();
    String queueInfoStr = baos.toString("UTF-8");
    Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}