Examples with CharsetDecoder java.nio.charset.CharsetDecoder used on opensource projects

Example 16 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project helios by spotify.

the class LoggingLogStreamFollower method followLog.

@Override
public void followLog(final JobId jobId, final String containerId, final Iterator<LogMessage> logStream) throws IOException {
    final Map<LogMessage.Stream, Decoder> streamDecoders = createStreamDecoders();
    final StringBuilder stringBuilder = new StringBuilder();
    LogMessage.Stream lastStream = null;
    try {
        while (logStream.hasNext()) {
            final LogMessage message = logStream.next();
            final ByteBuffer content = message.content();
            final LogMessage.Stream stream = message.stream();
            if (lastStream != null && lastStream != stream && stringBuilder.length() > 0) {
                log(lastStream, containerId, jobId, stringBuilder);
            }
            final Decoder decoder = streamDecoders.get(stream);
            final CharsetDecoder charsetDecoder = decoder.charsetDecoder;
            final ByteBuffer byteBuffer = decoder.byteBuffer;
            final CharBuffer charBuffer = decoder.charBuffer;
            while (content.hasRemaining()) {
                // Transfer as much of content into byteBuffer that we have room for
                byteBuffer.put(content);
                byteBuffer.flip();
                // Decode as much of byteBuffer into charBuffer that we can
                charsetDecoder.decode(byteBuffer, charBuffer, false);
                // The decoder might have left some partial byte sequences in the byteBuffer... Since we
                // don't have a ring buffer we should compact the buffer to not overflow.
                // We MUST NOT clear the byteBuffer since then we can lose those partial byte sequences.
                byteBuffer.compact();
                // Now start consuming the charBuffer
                charBuffer.flip();
                // Heuristic to avoid allocations... this will allocate too much memory if the charBuffer
                // contains any newlines or other special chars
                stringBuilder.ensureCapacity(charBuffer.remaining());
                while (charBuffer.hasRemaining()) {
                    final char c = charBuffer.get();
                    switch(c) {
                        case '\n':
                            log(stream, containerId, jobId, stringBuilder);
                            break;
                        default:
                            stringBuilder.append(c);
                    }
                }
                // This buffer is completely drained so we can reset it
                charBuffer.clear();
            }
            lastStream = stream;
        }
    } finally {
        if (lastStream != null && stringBuilder.length() > 0) {
            // Yes, we are not checking for any trailing bytes in the decoder byteBuffers here.  That
            // means that if the container wrote partial UTF-8 sequences before EOF they will be
            // discarded.
            log(lastStream, containerId, jobId, stringBuilder);
        }
    }
}

Example 17 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project cassandra by apache.

the class CBUtil method decodeString.

// Taken from Netty's ChannelBuffers.decodeString(). We need to use our own decoder to properly handle invalid
// UTF-8 sequences.  See CASSANDRA-8101 for more details.  This can be removed once https://github.com/netty/netty/pull/2999
// is resolved in a release used by Cassandra.
private static String decodeString(ByteBuffer src) throws CharacterCodingException {
    // the decoder needs to be reset every time we use it, hence the copy per thread
    CharsetDecoder theDecoder = TL_UTF8_DECODER.get();
    theDecoder.reset();
    CharBuffer dst = TL_CHAR_BUFFER.get();
    int capacity = (int) ((double) src.remaining() * theDecoder.maxCharsPerByte());
    if (dst == null) {
        capacity = Math.max(capacity, 4096);
        dst = CharBuffer.allocate(capacity);
        TL_CHAR_BUFFER.set(dst);
    } else {
        dst.clear();
        if (dst.capacity() < capacity) {
            dst = CharBuffer.allocate(capacity);
            TL_CHAR_BUFFER.set(dst);
        }
    }
    CoderResult cr = theDecoder.decode(src, dst, true);
    if (!cr.isUnderflow())
        cr.throwException();
    return dst.flip().toString();
}

Example 18 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project MusicDNA by harjot-oberai.

the class Mp4FtypBox method processData.

public void processData() throws CannotReadException {
    CharsetDecoder decoder = Charset.forName("ISO-8859-1").newDecoder();
    try {
        majorBrand = decoder.decode((ByteBuffer) dataBuffer.slice().limit(MAJOR_BRAND_LENGTH)).toString();
    } catch (CharacterCodingException cee) {
    //Ignore
    }
    dataBuffer.position(dataBuffer.position() + MAJOR_BRAND_LENGTH);
    majorBrandVersion = Utils.getIntBE(dataBuffer, dataBuffer.position(), (dataBuffer.position() + MAJOR_BRAND_VERSION_LENGTH - 1));
    dataBuffer.position(dataBuffer.position() + MAJOR_BRAND_VERSION_LENGTH);
    while ((dataBuffer.position() < dataBuffer.limit()) && (dataBuffer.limit() - dataBuffer.position() >= COMPATIBLE_BRAND_LENGTH)) {
        decoder.onMalformedInput(CodingErrorAction.REPORT);
        decoder.onMalformedInput(CodingErrorAction.REPORT);
        try {
            String brand = decoder.decode((ByteBuffer) dataBuffer.slice().limit(COMPATIBLE_BRAND_LENGTH)).toString();
            //Sometimes just extra groups of four nulls
            if (!brand.equals("")) {
                compatibleBrands.add(brand);
            }
        } catch (CharacterCodingException cee) {
        //Ignore    
        }
        dataBuffer.position(dataBuffer.position() + COMPATIBLE_BRAND_LENGTH);
    }
}

Example 19 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project j2objc by google.

the class CharsetDecoderTest method test_ByteArray_decode_with_offset_using_facade_method.

// http://code.google.com/p/android/issues/detail?id=4237
public void test_ByteArray_decode_with_offset_using_facade_method() throws Exception {
    CharsetDecoder decoder = Charset.forName("UTF-16").newDecoder();
    byte[] arr = encode("UTF-16", "Android");
    arr = prependByteToByteArray(arr, new Integer(1).byteValue());
    int offset = 1;
    CharBuffer outBuffer = decoder.decode(ByteBuffer.wrap(arr, offset, arr.length - offset));
    assertEquals("Android", outBuffer.toString().trim());
}

Example 20 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project j2objc by google.

the class CharsetDecoderTest method test_ByteArray_decode_with_offset.

// http://code.google.com/p/android/issues/detail?id=4237
public void test_ByteArray_decode_with_offset() throws Exception {
    CharsetDecoder decoder = Charset.forName("UTF-16").newDecoder();
    byte[] arr = encode("UTF-16", "Android");
    arr = prependByteToByteArray(arr, new Integer(1).byteValue());
    int offset = 1;
    ByteBuffer inBuffer = ByteBuffer.wrap(arr, offset, arr.length - offset).slice();
    CharBuffer outBuffer = CharBuffer.allocate(arr.length - offset);
    decoder.reset();
    CoderResult coderResult = decoder.decode(inBuffer, outBuffer, true);
    assertFalse(coderResult.toString(), coderResult.isError());
    decoder.flush(outBuffer);
    outBuffer.flip();
    assertEquals("Android", outBuffer.toString().trim());
}