Examples with BoundedInputStream org.apache.commons.io.input.BoundedInputStream used on opensource projects

Example 1 with BoundedInputStream

use of org.apache.commons.io.input.BoundedInputStream in project hadoop by apache.

the class ByteRangeInputStream method openInputStream.

@VisibleForTesting
protected InputStreamAndFileLength openInputStream(long startOffset) throws IOException {
    if (startOffset < 0) {
        throw new EOFException("Negative Position");
    }
    // Use the original url if no resolved url exists, eg. if
    // it's the first time a request is made.
    final boolean resolved = resolvedURL.getURL() != null;
    final URLOpener opener = resolved ? resolvedURL : originalURL;
    final HttpURLConnection connection = opener.connect(startOffset, resolved);
    resolvedURL.setURL(getResolvedUrl(connection));
    InputStream in = connection.getInputStream();
    final Long length;
    final Map<String, List<String>> headers = connection.getHeaderFields();
    if (isChunkedTransferEncoding(headers)) {
        // file length is not known
        length = null;
    } else {
        // for non-chunked transfer-encoding, get content-length
        final String cl = connection.getHeaderField(HttpHeaders.CONTENT_LENGTH);
        if (cl == null) {
            throw new IOException(HttpHeaders.CONTENT_LENGTH + " is missing: " + headers);
        }
        final long streamlength = Long.parseLong(cl);
        length = startOffset + streamlength;
        // Java has a bug with >2GB request streams.  It won't bounds check
        // the reads so the transfer blocks until the server times out
        in = new BoundedInputStream(in, streamlength);
    }
    return new InputStreamAndFileLength(length, in);
}

Example 2 with BoundedInputStream

use of org.apache.commons.io.input.BoundedInputStream in project disunity by ata4.

the class BundleReader method read.

public Bundle read() throws BundleException, IOException {
    bundle = new Bundle();
    in.position(0);
    BundleHeader header = bundle.header();
    in.readStruct(header);
    // check signature
    if (!header.hasValidSignature()) {
        throw new BundleException("Invalid signature");
    }
    List<BundleEntryInfo> entryInfos = bundle.entryInfos();
    if (header.compressedDataHeaderSize() > 0) {
        if (header.dataHeaderAtEndOfFile()) {
            in.position(header.completeFileSize() - header.compressedDataHeaderSize());
        }
        // build an input stream for the uncompressed data header
        InputStream headerIn = new BoundedInputStream(in.stream(), header.compressedDataHeaderSize());
        DataReader inData;
        switch(header.dataHeaderCompressionScheme()) {
            default:
            case 0:
                // Not compressed
                inData = DataReaders.forInputStream(headerIn);
            case 1:
                // LZMA
                inData = DataReaders.forInputStream(new CountingInputStream(new LzmaInputStream(headerIn)));
            case 3:
                // LZ4
                byte[] compressed = new byte[header.compressedDataHeaderSize()];
                byte[] decompressed = new byte[(int) header.dataHeaderSize()];
                headerIn.read(compressed);
                LZ4JavaSafeFastDecompressor.INSTANCE.decompress(compressed, decompressed);
                inData = DataReaders.forByteBuffer(ByteBuffer.wrap(decompressed));
        }
        // Block info: not captured for now
        {
            // 16 bytes unknown
            byte[] unknown = new byte[16];
            inData.readBytes(unknown);
            int storageBlocks = inData.readInt();
            for (int i = 0; i < storageBlocks; ++i) {
                inData.readUnsignedInt();
                inData.readUnsignedInt();
                inData.readUnsignedShort();
            }
        }
        int files = inData.readInt();
        for (int i = 0; i < files; i++) {
            BundleEntryInfo entryInfo = new BundleEntryInfoFS();
            inData.readStruct(entryInfo);
            entryInfos.add(entryInfo);
        }
    } else {
        // raw or web header
        long dataHeaderSize = header.dataHeaderSize();
        if (dataHeaderSize == 0) {
            // old stream versions don't store the data header size, so use a large
            // fixed number instead
            dataHeaderSize = 4096;
        }
        InputStream is = dataInputStream(0, dataHeaderSize);
        DataReader inData = DataReaders.forInputStream(is);
        int files = inData.readInt();
        for (int i = 0; i < files; i++) {
            BundleEntryInfo entryInfo = new BundleEntryInfo();
            inData.readStruct(entryInfo);
            entryInfos.add(entryInfo);
        }
    }
    // sort entries by offset so that they're in the order in which they
    // appear in the file, which is convenient for compressed bundles
    entryInfos.sort((a, b) -> Long.compare(a.offset(), b.offset()));
    List<BundleEntry> entries = bundle.entries();
    entryInfos.forEach(entryInfo -> {
        entries.add(new BundleInternalEntry(entryInfo, this::inputStreamForEntry));
    });
    return bundle;
}

Example 3 with BoundedInputStream

use of org.apache.commons.io.input.BoundedInputStream in project jmeter by apache.

the class HTTPHC4Impl method sendEntityData.

// TODO merge put and post methods as far as possible.
// e.g. post checks for multipart form/files, and if not, invokes sendData(HttpEntityEnclosingRequestBase)
/**
     * Creates the entity data to be sent.
     * <p>
     * If there is a file entry with a non-empty MIME type we use that to
     * set the request Content-Type header, otherwise we default to whatever
     * header is present from a Header Manager.
     * <p>
     * If the content charset {@link #getContentEncoding()} is null or empty 
     * we use the HC4 default provided by {@link HTTP#DEF_CONTENT_CHARSET} which is
     * ISO-8859-1.
     * 
     * @param entity to be processed, e.g. PUT or PATCH
     * @return the entity content, may be empty
     * @throws  UnsupportedEncodingException for invalid charset name
     * @throws IOException cannot really occur for ByteArrayOutputStream methods
     */
protected String sendEntityData(HttpEntityEnclosingRequestBase entity) throws IOException {
    boolean hasEntityBody = false;
    final HTTPFileArg[] files = getHTTPFiles();
    // Allow the mimetype of the file to control the content type
    // This is not obvious in GUI if you are not uploading any files,
    // but just sending the content of nameless parameters
    final HTTPFileArg file = files.length > 0 ? files[0] : null;
    String contentTypeValue = null;
    if (file != null && file.getMimeType() != null && file.getMimeType().length() > 0) {
        contentTypeValue = file.getMimeType();
        // we provide the MIME type here
        entity.setHeader(HEADER_CONTENT_TYPE, contentTypeValue);
    }
    // Check for local contentEncoding (charset) override; fall back to default for content body
    // we do this here rather so we can use the same charset to retrieve the data
    final String charset = getContentEncoding(HTTP.DEF_CONTENT_CHARSET.name());
    if (!hasArguments() && getSendFileAsPostBody()) {
        hasEntityBody = true;
        // If getSendFileAsPostBody returned true, it's sure that file is not null
        File reservedFile = FileServer.getFileServer().getResolvedFile(files[0].getPath());
        // no need for content-type here
        FileEntity fileRequestEntity = new FileEntity(reservedFile);
        entity.setEntity(fileRequestEntity);
    } else // just send all the values as the entity body
    if (getSendParameterValuesAsPostBody()) {
        hasEntityBody = true;
        // Just append all the parameter values, and use that as the entity body
        Arguments arguments = getArguments();
        StringBuilder entityBodyContent = new StringBuilder(arguments.getArgumentCount() * 15);
        for (JMeterProperty jMeterProperty : arguments) {
            HTTPArgument arg = (HTTPArgument) jMeterProperty.getObjectValue();
            // Note: if "Encoded?" is not selected, arg.getEncodedValue is equivalent to arg.getValue
            if (charset != null) {
                entityBodyContent.append(arg.getEncodedValue(charset));
            } else {
                entityBodyContent.append(arg.getEncodedValue());
            }
        }
        StringEntity requestEntity = new StringEntity(entityBodyContent.toString(), charset);
        entity.setEntity(requestEntity);
    }
    // Check if we have any content to send for body
    if (hasEntityBody) {
        // If the request entity is repeatable, we can send it first to
        // our own stream, so we can return it
        final HttpEntity entityEntry = entity.getEntity();
        // Buffer to hold the entity body
        StringBuilder entityBody = null;
        if (entityEntry.isRepeatable()) {
            entityBody = new StringBuilder(1000);
            // FIXME Charset
            try (InputStream in = entityEntry.getContent();
                InputStream bounded = new BoundedInputStream(in, MAX_BODY_RETAIN_SIZE)) {
                entityBody.append(IOUtils.toString(bounded));
            }
            if (entityEntry.getContentLength() > MAX_BODY_RETAIN_SIZE) {
                entityBody.append("<actual file content shortened>");
            }
        } else {
            entityBody = new StringBuilder(65);
            // this probably cannot happen
            entityBody.append("<RequestEntity was not repeatable, cannot view what was sent>");
        }
        return entityBody.toString();
    }
    // may be the empty string
    return "";
}

Example 4 with BoundedInputStream

use of org.apache.commons.io.input.BoundedInputStream in project disunity by ata4.

the class BundleReader method dataInputStream.

private InputStream dataInputStream(long offset, long size) throws IOException {
    InputStream is;
    // use LZMA stream if the bundle is compressed
    if (bundle.header().compressed()) {
        // create initial input stream if required
        if (lzma == null) {
            lzma = lzmaInputStream();
        }
        // recreate stream if the offset is behind
        long lzmaOffset = lzma.getByteCount();
        if (lzmaOffset > offset) {
            lzma.close();
            lzma = lzmaInputStream();
        }
        // skip forward if required
        if (lzmaOffset < offset) {
            lzma.skip(offset - lzmaOffset);
        }
        is = lzma;
    } else {
        in.position(bundle.header().headerSize() + offset);
        is = in.stream();
    }
    return new BoundedInputStream(is, size);
}

Example 5 with BoundedInputStream

use of org.apache.commons.io.input.BoundedInputStream in project k-9 by k9mail.

the class MessageExtractor method getTextFromTextPart.

private static String getTextFromTextPart(Part part, Body body, String mimeType, long textSizeLimit) throws IOException, MessagingException {
    /*
         * We've got a text part, so let's see if it needs to be processed further.
         */
    String charset = getHeaderParameter(part.getContentType(), "charset");
    /*
         * determine the charset from HTML message.
         */
    if (isSameMimeType(mimeType, "text/html") && charset == null) {
        InputStream in = MimeUtility.decodeBody(body);
        try {
            byte[] buf = new byte[256];
            in.read(buf, 0, buf.length);
            String str = new String(buf, "US-ASCII");
            if (str.isEmpty()) {
                return "";
            }
            Pattern p = Pattern.compile("<meta http-equiv=\"?Content-Type\"? content=\"text/html; charset=(.+?)\">", Pattern.CASE_INSENSITIVE);
            Matcher m = p.matcher(str);
            if (m.find()) {
                charset = m.group(1);
            }
        } finally {
            try {
                MimeUtility.closeInputStreamWithoutDeletingTemporaryFiles(in);
            } catch (IOException e) {
            /* ignore */
            }
        }
    }
    charset = fixupCharset(charset, getMessageFromPart(part));
    /*
         * Now we read the part into a buffer for further processing. Because
         * the stream is now wrapped we'll remove any transfer encoding at this point.
         */
    InputStream in = MimeUtility.decodeBody(body);
    InputStream possiblyLimitedIn = textSizeLimit != NO_TEXT_SIZE_LIMIT ? new BoundedInputStream(in, textSizeLimit) : in;
    try {
        return CharsetSupport.readToString(possiblyLimitedIn, charset);
    } finally {
        try {
            MimeUtility.closeInputStreamWithoutDeletingTemporaryFiles(in);
        } catch (IOException e) {
        /* Ignore */
        }
    }
}