Examples with CharsetDecoder java.nio.charset.CharsetDecoder used on opensource projects

Example 1 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project voltdb by VoltDB.

the class CharsetUtil method decoder.

/**
     * Returns a cached thread-local {@link CharsetDecoder} for the specified {@link Charset}.
     *
     * @param charset The specified charset
     * @return The decoder for the specified <code>charset</code>
     */
public static CharsetDecoder decoder(Charset charset) {
    checkNotNull(charset, "charset");
    Map<Charset, CharsetDecoder> map = InternalThreadLocalMap.get().charsetDecoderCache();
    CharsetDecoder d = map.get(charset);
    if (d != null) {
        d.reset().onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE);
        return d;
    }
    d = decoder(charset, CodingErrorAction.REPLACE, CodingErrorAction.REPLACE);
    map.put(charset, d);
    return d;
}

Example 2 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project voltdb by VoltDB.

the class CharsetUtil method decoder.

/**
     * Returns a new {@link CharsetDecoder} for the {@link Charset} with specified error actions.
     *
     * @param charset The specified charset
     * @param malformedInputAction The decoder's action for malformed-input errors
     * @param unmappableCharacterAction The decoder's action for unmappable-character errors
     * @return The decoder for the specified <code>charset</code>
     */
public static CharsetDecoder decoder(Charset charset, CodingErrorAction malformedInputAction, CodingErrorAction unmappableCharacterAction) {
    checkNotNull(charset, "charset");
    CharsetDecoder d = charset.newDecoder();
    d.onMalformedInput(malformedInputAction).onUnmappableCharacter(unmappableCharacterAction);
    return d;
}

Example 3 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project voltdb by VoltDB.

the class ByteBufUtil method decodeString.

static String decodeString(ByteBuf src, int readerIndex, int len, Charset charset) {
    if (len == 0) {
        return StringUtil.EMPTY_STRING;
    }
    final CharsetDecoder decoder = CharsetUtil.decoder(charset);
    final int maxLength = (int) ((double) len * decoder.maxCharsPerByte());
    CharBuffer dst = CHAR_BUFFERS.get();
    if (dst.length() < maxLength) {
        dst = CharBuffer.allocate(maxLength);
        if (maxLength <= MAX_CHAR_BUFFER_SIZE) {
            CHAR_BUFFERS.set(dst);
        }
    } else {
        dst.clear();
    }
    if (src.nioBufferCount() == 1) {
        // Use internalNioBuffer(...) to reduce object creation.
        decodeString(decoder, src.internalNioBuffer(readerIndex, len), dst);
    } else {
        // We use a heap buffer as CharsetDecoder is most likely able to use a fast-path if src and dst buffers
        // are both backed by a byte array.
        ByteBuf buffer = src.alloc().heapBuffer(len);
        try {
            buffer.writeBytes(src, readerIndex, len);
            // Use internalNioBuffer(...) to reduce object creation.
            decodeString(decoder, buffer.internalNioBuffer(0, len), dst);
        } finally {
            // Release the temporary buffer again.
            buffer.release();
        }
    }
    return dst.flip().toString();
}

Example 4 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project jdk8u_jdk by JetBrains.

the class NIOJISAutoDetectTest method detectingCharset.

private static String detectingCharset(byte[] bytes) throws Exception {
    //----------------------------------------------------------------
    // Test special public methods of CharsetDecoder while we're here
    //----------------------------------------------------------------
    CharsetDecoder cd = Charset.forName("JISAutodetect").newDecoder();
    check(cd.isAutoDetecting(), "isAutodecting()");
    check(!cd.isCharsetDetected(), "isCharsetDetected");
    cd.decode(ByteBuffer.wrap(new byte[] { (byte) 'A' }));
    check(!cd.isCharsetDetected(), "isCharsetDetected");
    try {
        cd.detectedCharset();
        fail("no IllegalStateException");
    } catch (IllegalStateException e) {
    }
    cd.decode(ByteBuffer.wrap(bytes));
    check(cd.isCharsetDetected(), "isCharsetDetected");
    Charset cs = cd.detectedCharset();
    check(cs != null, "cs != null");
    check(!cs.newDecoder().isAutoDetecting(), "isAutodetecting()");
    return cs.name();
}

Example 5 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project lucene-solr by apache.

the class SolrRequestParsers method parseFormDataContent.

/**
   * Given a url-encoded form from POST content (as InputStream), map it into the given map.
   * The given InputStream should be buffered!
   * @param postContent to be parsed
   * @param charset to be used to decode resulting bytes after %-decoding
   * @param map place all parameters in this map
   */
@SuppressWarnings({ "fallthrough", "resource" })
static long parseFormDataContent(final InputStream postContent, final long maxLen, Charset charset, final Map<String, String[]> map, boolean supportCharsetParam) throws IOException {
    CharsetDecoder charsetDecoder = supportCharsetParam ? null : getCharsetDecoder(charset);
    final LinkedList<Object> buffer = supportCharsetParam ? new LinkedList<>() : null;
    long len = 0L, keyPos = 0L, valuePos = 0L;
    final ByteArrayOutputStream keyStream = new ByteArrayOutputStream(), valueStream = new ByteArrayOutputStream();
    ByteArrayOutputStream currentStream = keyStream;
    for (; ; ) {
        int b = postContent.read();
        switch(b) {
            // end of stream
            case -1:
            case // separator
            '&':
                if (keyStream.size() > 0) {
                    final byte[] keyBytes = keyStream.toByteArray(), valueBytes = valueStream.toByteArray();
                    if (Arrays.equals(keyBytes, INPUT_ENCODING_BYTES)) {
                        // we found a charset declaration in the raw bytes
                        if (charsetDecoder != null) {
                            throw new SolrException(ErrorCode.BAD_REQUEST, supportCharsetParam ? ("Query string invalid: duplicate '" + INPUT_ENCODING_KEY + "' (input encoding) key.") : ("Key '" + INPUT_ENCODING_KEY + "' (input encoding) cannot " + "be used in POSTed application/x-www-form-urlencoded form data. " + "To set the input encoding of POSTed form data, use the " + "'Content-Type' header and provide a charset!"));
                        }
                        // decode the charset from raw bytes
                        charset = Charset.forName(decodeChars(valueBytes, keyPos, getCharsetDecoder(CHARSET_US_ASCII)));
                        charsetDecoder = getCharsetDecoder(charset);
                        // finally decode all buffered tokens
                        decodeBuffer(buffer, map, charsetDecoder);
                    } else if (charsetDecoder == null) {
                        // we have no charset decoder until now, buffer the keys / values for later processing:
                        buffer.add(keyBytes);
                        buffer.add(Long.valueOf(keyPos));
                        buffer.add(valueBytes);
                        buffer.add(Long.valueOf(valuePos));
                    } else {
                        // we already have a charsetDecoder, so we can directly decode without buffering:
                        final String key = decodeChars(keyBytes, keyPos, charsetDecoder), value = decodeChars(valueBytes, valuePos, charsetDecoder);
                        MultiMapSolrParams.addParam(key.trim(), value, map);
                    }
                } else if (valueStream.size() > 0) {
                    throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded invalid: missing key");
                }
                keyStream.reset();
                valueStream.reset();
                keyPos = valuePos = len + 1;
                currentStream = keyStream;
                break;
            case // space replacement
            '+':
                currentStream.write(' ');
                break;
            case // escape
            '%':
                final int upper = digit16(b = postContent.read());
                len++;
                final int lower = digit16(b = postContent.read());
                len++;
                currentStream.write(((upper << 4) + lower));
                break;
            case // kv separator
            '=':
                if (currentStream == keyStream) {
                    valuePos = len + 1;
                    currentStream = valueStream;
                    break;
                }
            // fall-through
            default:
                currentStream.write(b);
        }
        if (b == -1) {
            break;
        }
        len++;
        if (len > maxLen) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content exceeds upload limit of " + (maxLen / 1024L) + " KB");
        }
    }
    // if we have not seen a charset declaration, decode the buffer now using the default one (UTF-8 or given via Content-Type):
    if (buffer != null && !buffer.isEmpty()) {
        assert charsetDecoder == null;
        decodeBuffer(buffer, map, getCharsetDecoder(charset));
    }
    return len;
}