Examples with FileInputStream java.io.FileInputStream used on opensource projects

Example 96 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class TestTextOutputFormat method testCompress.

/**
   * test compressed file
   * @throws IOException
   */
@Test
public void testCompress() throws IOException {
    JobConf job = new JobConf();
    job.set(JobContext.TASK_ATTEMPT_ID, attempt);
    job.set(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS, "true");
    FileOutputFormat.setOutputPath(job, workDir.getParent().getParent());
    FileOutputFormat.setWorkOutputPath(job, workDir);
    FileSystem fs = workDir.getFileSystem(job);
    if (!fs.mkdirs(workDir)) {
        fail("Failed to create output directory");
    }
    String file = "test_compress.txt";
    // A reporter that does nothing
    Reporter reporter = Reporter.NULL;
    TextOutputFormat<Object, Object> theOutputFormat = new TextOutputFormat<Object, Object>();
    RecordWriter<Object, Object> theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter);
    Text key1 = new Text("key1");
    Text key2 = new Text("key2");
    Text val1 = new Text("val1");
    Text val2 = new Text("val2");
    NullWritable nullWritable = NullWritable.get();
    try {
        theRecordWriter.write(key1, val1);
        theRecordWriter.write(null, nullWritable);
        theRecordWriter.write(null, val1);
        theRecordWriter.write(nullWritable, val2);
        theRecordWriter.write(key2, nullWritable);
        theRecordWriter.write(key1, null);
        theRecordWriter.write(null, null);
        theRecordWriter.write(key2, val2);
    } finally {
        theRecordWriter.close(reporter);
    }
    StringBuffer expectedOutput = new StringBuffer();
    expectedOutput.append(key1).append("\t").append(val1).append("\n");
    expectedOutput.append(val1).append("\n");
    expectedOutput.append(val2).append("\n");
    expectedOutput.append(key2).append("\n");
    expectedOutput.append(key1).append("\n");
    expectedOutput.append(key2).append("\t").append(val2).append("\n");
    DefaultCodec codec = new DefaultCodec();
    codec.setConf(job);
    Path expectedFile = new Path(workDir, file + codec.getDefaultExtension());
    final FileInputStream istream = new FileInputStream(expectedFile.toString());
    CompressionInputStream cistream = codec.createInputStream(istream);
    LineReader reader = new LineReader(cistream);
    String output = "";
    Text out = new Text();
    while (reader.readLine(out) > 0) {
        output += out;
        output += "\n";
    }
    reader.close();
    assertEquals(expectedOutput.toString(), output);
}

Example 97 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class SLSUtils method parseNodesFromSLSTrace.

/**
   * parse the sls trace file, return each host name
   */
public static Set<String> parseNodesFromSLSTrace(String jobTrace) throws IOException {
    Set<String> nodeSet = new HashSet<String>();
    JsonFactory jsonF = new JsonFactory();
    ObjectMapper mapper = new ObjectMapper();
    Reader input = new InputStreamReader(new FileInputStream(jobTrace), "UTF-8");
    try {
        Iterator<Map> i = mapper.readValues(jsonF.createParser(input), Map.class);
        while (i.hasNext()) {
            Map jsonE = i.next();
            List tasks = (List) jsonE.get("job.tasks");
            for (Object o : tasks) {
                Map jsonTask = (Map) o;
                String hostname = jsonTask.get("container.host").toString();
                nodeSet.add(hostname);
            }
        }
    } finally {
        input.close();
    }
    return nodeSet;
}

Example 98 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class TestApplicationHistoryServer method testHostedUIs.

@Test(timeout = 240000)
public void testHostedUIs() throws Exception {
    ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
    Configuration config = new YarnConfiguration();
    config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, MemoryTimelineStore.class, TimelineStore.class);
    config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, MemoryTimelineStateStore.class, TimelineStateStore.class);
    config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0");
    final String UI1 = "UI1";
    String connFileStr = "";
    File diskFile = new File("./pom.xml");
    String diskFileStr = readInputStream(new FileInputStream(diskFile));
    try {
        config.set(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES, UI1);
        config.set(YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + UI1, "/" + UI1);
        config.set(YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX + UI1, "./");
        historyServer.init(config);
        historyServer.start();
        URL url = new URL("http://localhost:" + historyServer.getPort() + "/" + UI1 + "/pom.xml");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.connect();
        assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
        connFileStr = readInputStream(conn.getInputStream());
    } finally {
        historyServer.stop();
    }
    assertEquals("Web file contents should be the same as on disk contents", diskFileStr, connFileStr);
}

Example 99 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class ProcessIdFileReader method getProcessId.

/**
   * Get the process id from specified file path.
   * Parses each line to find a valid number
   * and returns the first one found.
   * @return Process Id if obtained from path specified else null
   * @throws IOException
   */
public static String getProcessId(Path path) throws IOException {
    if (path == null) {
        throw new IOException("Trying to access process id from a null path");
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Accessing pid from pid file " + path);
    }
    String processId = null;
    BufferedReader bufReader = null;
    try {
        File file = new File(path.toString());
        if (file.exists()) {
            FileInputStream fis = new FileInputStream(file);
            bufReader = new BufferedReader(new InputStreamReader(fis, "UTF-8"));
            while (true) {
                String line = bufReader.readLine();
                if (line == null) {
                    break;
                }
                String temp = line.trim();
                if (!temp.isEmpty()) {
                    if (Shell.WINDOWS) {
                        // line that parses successfully as a container ID.
                        try {
                            ContainerId.fromString(temp);
                            processId = temp;
                            break;
                        } catch (Exception e) {
                        // do nothing
                        }
                    } else {
                        // Otherwise, find first line containing a numeric pid.
                        try {
                            long pid = Long.parseLong(temp);
                            if (pid > 0) {
                                processId = temp;
                                break;
                            }
                        } catch (Exception e) {
                        // do nothing
                        }
                    }
                }
            }
        }
    } finally {
        if (bufReader != null) {
            bufReader.close();
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Got pid " + (processId != null ? processId : "null") + " from path " + path);
    }
    return processId;
}

Example 100 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class NMWebServices method getLogs.

/**
   * Returns the contents of a container's log file in plain text. 
   *
   * Only works for containers that are still in the NodeManager's memory, so
   * logs are no longer available after the corresponding application is no
   * longer running.
   * 
   * @param containerIdStr
   *    The container ID
   * @param filename
   *    The name of the log file
   * @param format
   *    The content type
   * @param size
   *    the size of the log file
   * @return
   *    The contents of the container's log file
   */
@GET
@Path("/containerlogs/{containerid}/{filename}")
@Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 })
@Public
@Unstable
public Response getLogs(@PathParam(YarnWebServiceParams.CONTAINER_ID) final String containerIdStr, @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size) {
    ContainerId tempContainerId;
    try {
        tempContainerId = ContainerId.fromString(containerIdStr);
    } catch (IllegalArgumentException ex) {
        return Response.status(Status.BAD_REQUEST).build();
    }
    final ContainerId containerId = tempContainerId;
    boolean tempIsRunning = false;
    // check what is the status for container
    try {
        Container container = nmContext.getContainers().get(containerId);
        tempIsRunning = (container.getContainerState() == ContainerState.RUNNING);
    } catch (Exception ex) {
        // assume the container has already finished.
        if (LOG.isDebugEnabled()) {
            LOG.debug("Can not find the container:" + containerId + " in this node.");
        }
    }
    final boolean isRunning = tempIsRunning;
    File logFile = null;
    try {
        logFile = ContainerLogsUtils.getContainerLogFile(containerId, filename, request.getRemoteUser(), nmContext);
    } catch (NotFoundException ex) {
        if (redirectWSUrl == null || redirectWSUrl.isEmpty()) {
            return Response.status(Status.NOT_FOUND).entity(ex.getMessage()).build();
        }
        // redirect the request to the configured log server
        String redirectURI = "/containers/" + containerIdStr + "/logs/" + filename;
        return createRedirectResponse(request, redirectWSUrl, redirectURI);
    } catch (YarnException ex) {
        return Response.serverError().entity(ex.getMessage()).build();
    }
    final long bytes = parseLongParam(size);
    final String lastModifiedTime = Times.format(logFile.lastModified());
    final String outputFileName = filename;
    String contentType = WebAppUtils.getDefaultLogContentType();
    if (format != null && !format.isEmpty()) {
        contentType = WebAppUtils.getSupportedLogContentType(format);
        if (contentType == null) {
            String errorMessage = "The valid values for the parameter : format " + "are " + WebAppUtils.listSupportedLogContentType();
            return Response.status(Status.BAD_REQUEST).entity(errorMessage).build();
        }
    }
    try {
        final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(containerIdStr, logFile, nmContext);
        final long fileLength = logFile.length();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream os) throws IOException, WebApplicationException {
                try {
                    int bufferSize = 65536;
                    byte[] buf = new byte[bufferSize];
                    LogToolUtils.outputContainerLog(containerId.toString(), nmContext.getNodeId().toString(), outputFileName, fileLength, bytes, lastModifiedTime, fis, os, buf, ContainerLogAggregationType.LOCAL);
                    StringBuilder sb = new StringBuilder();
                    String endOfFile = "End of LogType:" + outputFileName;
                    sb.append(endOfFile + ".");
                    if (isRunning) {
                        sb.append("This log file belongs to a running container (" + containerIdStr + ") and so may not be complete." + "\n");
                    } else {
                        sb.append("\n");
                    }
                    sb.append(StringUtils.repeat("*", endOfFile.length() + 50) + "\n\n");
                    os.write(sb.toString().getBytes(Charset.forName("UTF-8")));
                    // If we have aggregated logs for this container,
                    // output the aggregation logs as well.
                    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
                    Application app = nmContext.getApplications().get(appId);
                    String appOwner = app == null ? null : app.getUser();
                    try {
                        LogToolUtils.outputAggregatedContainerLog(nmContext.getConf(), appId, appOwner, containerId.toString(), nmContext.getNodeId().toString(), outputFileName, bytes, os, buf);
                    } catch (Exception ex) {
                        // Something wrong when we try to access the aggregated log.
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Can not access the aggregated log for " + "the container:" + containerId);
                            LOG.debug(ex.getMessage());
                        }
                    }
                } finally {
                    IOUtils.closeQuietly(fis);
                }
            }
        };
        ResponseBuilder resp = Response.ok(stream);
        resp.header("Content-Type", contentType + "; " + JettyUtils.UTF_8);
        // Sending the X-Content-Type-Options response header with the value
        // nosniff will prevent Internet Explorer from MIME-sniffing a response
        // away from the declared content-type.
        resp.header("X-Content-Type-Options", "nosniff");
        return resp.build();
    } catch (IOException ex) {
        return Response.serverError().entity(ex.getMessage()).build();
    }
}