Examples with FSDataInputStream org.apache.hadoop.fs.FSDataInputStream used on opensource projects

Example 91 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class JsonSerDeser method load.

/**
   * Load from a Hadoop filesystem
   * @param fs filesystem
   * @param path path
   * @return a loaded CD
   * @throws IOException IO problems
   * @throws EOFException if not enough bytes were read in
   * @throws JsonParseException parse problems
   * @throws JsonMappingException O/J mapping problems
   */
public T load(FileSystem fs, Path path) throws IOException, JsonParseException, JsonMappingException {
    FileStatus status = fs.getFileStatus(path);
    long len = status.getLen();
    byte[] b = new byte[(int) len];
    FSDataInputStream dataInputStream = fs.open(path);
    int count = dataInputStream.read(b);
    if (count != len) {
        throw new EOFException(path.toString() + ": read finished prematurely");
    }
    return fromBytes(path.toString(), b);
}

Example 92 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class ContainerLaunch method handleContainerExitWithFailure.

/**
   * Tries to tail and fetch TAIL_SIZE_IN_BYTES of data from the error log.
   * ErrorLog filename is not fixed and depends upon app, hence file name
   * pattern is used.
   * @param containerID
   * @param ret
   * @param containerLogDir
   * @param diagnosticInfo
   */
@SuppressWarnings("unchecked")
protected void handleContainerExitWithFailure(ContainerId containerID, int ret, Path containerLogDir, StringBuilder diagnosticInfo) {
    LOG.warn(diagnosticInfo);
    String errorFileNamePattern = conf.get(YarnConfiguration.NM_CONTAINER_STDERR_PATTERN, YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_PATTERN);
    FSDataInputStream errorFileIS = null;
    try {
        FileSystem fileSystem = FileSystem.getLocal(conf).getRaw();
        FileStatus[] errorFileStatuses = fileSystem.globStatus(new Path(containerLogDir, errorFileNamePattern));
        if (errorFileStatuses != null && errorFileStatuses.length != 0) {
            long tailSizeInBytes = conf.getLong(YarnConfiguration.NM_CONTAINER_STDERR_BYTES, YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_BYTES);
            Path errorFile = errorFileStatuses[0].getPath();
            long fileSize = errorFileStatuses[0].getLen();
            // modified file, and also append the file names in the diagnosticInfo
            if (errorFileStatuses.length > 1) {
                String[] errorFileNames = new String[errorFileStatuses.length];
                long latestModifiedTime = errorFileStatuses[0].getModificationTime();
                errorFileNames[0] = errorFileStatuses[0].getPath().getName();
                for (int i = 1; i < errorFileStatuses.length; i++) {
                    errorFileNames[i] = errorFileStatuses[i].getPath().getName();
                    if (errorFileStatuses[i].getModificationTime() > latestModifiedTime) {
                        latestModifiedTime = errorFileStatuses[i].getModificationTime();
                        errorFile = errorFileStatuses[i].getPath();
                        fileSize = errorFileStatuses[i].getLen();
                    }
                }
                diagnosticInfo.append("Error files: ").append(StringUtils.join(", ", errorFileNames)).append(".\n");
            }
            long startPosition = (fileSize < tailSizeInBytes) ? 0 : fileSize - tailSizeInBytes;
            int bufferSize = (int) ((fileSize < tailSizeInBytes) ? fileSize : tailSizeInBytes);
            byte[] tailBuffer = new byte[bufferSize];
            errorFileIS = fileSystem.open(errorFile);
            errorFileIS.readFully(startPosition, tailBuffer);
            diagnosticInfo.append("Last ").append(tailSizeInBytes).append(" bytes of ").append(errorFile.getName()).append(" :\n").append(new String(tailBuffer, StandardCharsets.UTF_8));
        }
    } catch (IOException e) {
        LOG.error("Failed to get tail of the container's error log file", e);
    } finally {
        IOUtils.cleanup(LOG, errorFileIS);
    }
    this.dispatcher.getEventHandler().handle(new ContainerExitEvent(containerID, ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret, diagnosticInfo.toString()));
}

Example 93 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class NativeAzureFileSystem method open.

@Override
public FSDataInputStream open(Path f, int bufferSize) throws FileNotFoundException, IOException {
    LOG.debug("Opening file: {}", f.toString());
    Path absolutePath = makeAbsolute(f);
    performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ.toString(), "read");
    String key = pathToKey(absolutePath);
    FileMetadata meta = null;
    try {
        meta = store.retrieveMetadata(key);
    } catch (Exception ex) {
        Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
        if (innerException instanceof StorageException && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
            throw new FileNotFoundException(String.format("%s is not found", key));
        }
        throw ex;
    }
    if (meta == null) {
        throw new FileNotFoundException(f.toString());
    }
    if (meta.isDir()) {
        throw new FileNotFoundException(f.toString() + " is a directory not a file.");
    }
    DataInputStream inputStream = null;
    try {
        inputStream = store.retrieve(key);
    } catch (Exception ex) {
        Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex);
        if (innerException instanceof StorageException && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
            throw new FileNotFoundException(String.format("%s is not found", key));
        }
        throw ex;
    }
    return new FSDataInputStream(new BufferedFSInputStream(new NativeAzureFsInputStream(inputStream, key, meta.getLength()), bufferSize));
}

Example 94 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class NativeAzureFileSystemBaseTest method readString.

private String readString(FileSystem fs, Path testFile) throws IOException {
    FSDataInputStream inputStream = fs.open(testFile);
    String ret = readString(inputStream);
    inputStream.close();
    return ret;
}

Example 95 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class StreamInputFormat method getRecordReader.

@SuppressWarnings("unchecked")
public RecordReader<Text, Text> getRecordReader(final InputSplit genericSplit, JobConf job, Reporter reporter) throws IOException {
    String c = job.get("stream.recordreader.class");
    if (c == null || c.indexOf("LineRecordReader") >= 0) {
        return super.getRecordReader(genericSplit, job, reporter);
    }
    // handling non-standard record reader (likely StreamXmlRecordReader) 
    FileSplit split = (FileSplit) genericSplit;
    LOG.info("getRecordReader start.....split=" + split);
    reporter.setStatus(split.toString());
    // Open the file and seek to the start of the split
    FileSystem fs = split.getPath().getFileSystem(job);
    FSDataInputStream in = fs.open(split.getPath());
    // Factory dispatch based on available params..
    Class readerClass;
    {
        readerClass = StreamUtil.goodClassOrNull(job, c, null);
        if (readerClass == null) {
            throw new RuntimeException("Class not found: " + c);
        }
    }
    Constructor ctor;
    try {
        ctor = readerClass.getConstructor(new Class[] { FSDataInputStream.class, FileSplit.class, Reporter.class, JobConf.class, FileSystem.class });
    } catch (NoSuchMethodException nsm) {
        throw new RuntimeException(nsm);
    }
    RecordReader<Text, Text> reader;
    try {
        reader = (RecordReader<Text, Text>) ctor.newInstance(new Object[] { in, split, reporter, job, fs });
    } catch (Exception nsm) {
        throw new RuntimeException(nsm);
    }
    return reader;
}