Examples with CharsetDecoder java.nio.charset.CharsetDecoder used on opensource projects

Example 31 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project lucene-solr by apache.

the class TestCompactLabelToOrdinal method testL2O.

@Test
public void testL2O() throws Exception {
    LabelToOrdinal map = new LabelToOrdinalMap();
    CompactLabelToOrdinal compact = new CompactLabelToOrdinal(2000000, 0.15f, 3);
    final int n = atLeast(10 * 1000);
    final int numUniqueValues = 50 * 1000;
    String[] uniqueValues = new String[numUniqueValues];
    byte[] buffer = new byte[50];
    Random random = random();
    for (int i = 0; i < numUniqueValues; ) {
        random.nextBytes(buffer);
        int size = 1 + random.nextInt(buffer.length);
        // This test is turning random bytes into a string,
        // this is asking for trouble.
        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onUnmappableCharacter(CodingErrorAction.REPLACE).onMalformedInput(CodingErrorAction.REPLACE);
        uniqueValues[i] = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
        // we cannot have empty path components, so eliminate all prefix as well
        // as middle consecutive delimiter chars.
        uniqueValues[i] = uniqueValues[i].replaceAll("/+", "/");
        if (uniqueValues[i].startsWith("/")) {
            uniqueValues[i] = uniqueValues[i].substring(1);
        }
        if (uniqueValues[i].indexOf(CompactLabelToOrdinal.TERMINATOR_CHAR) == -1) {
            i++;
        }
    }
    Path tmpDir = createTempDir("testLableToOrdinal");
    Path f = tmpDir.resolve("CompactLabelToOrdinalTest.tmp");
    int flushInterval = 10;
    for (int i = 0; i < n; i++) {
        if (i > 0 && i % flushInterval == 0) {
            compact.flush(f);
            compact = CompactLabelToOrdinal.open(f, 0.15f, 3);
            Files.delete(f);
            if (flushInterval < (n / 10)) {
                flushInterval *= 10;
            }
        }
        int index = random.nextInt(numUniqueValues);
        FacetLabel label;
        String s = uniqueValues[index];
        if (s.length() == 0) {
            label = new FacetLabel();
        } else {
            label = new FacetLabel(s.split("/"));
        }
        int ord1 = map.getOrdinal(label);
        int ord2 = compact.getOrdinal(label);
        assertEquals(ord1, ord2);
        if (ord1 == LabelToOrdinal.INVALID_ORDINAL) {
            ord1 = compact.getNextOrdinal();
            map.addLabel(label, ord1);
            compact.addLabel(label, ord1);
        }
    }
    for (int i = 0; i < numUniqueValues; i++) {
        FacetLabel label;
        String s = uniqueValues[i];
        if (s.length() == 0) {
            label = new FacetLabel();
        } else {
            label = new FacetLabel(s.split("/"));
        }
        int ord1 = map.getOrdinal(label);
        int ord2 = compact.getOrdinal(label);
        assertEquals(ord1, ord2);
    }
}

Example 32 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project undertow by undertow-io.

the class AsyncReceiverImpl method receivePartialString.

@Override
public void receivePartialString(final PartialStringCallback callback, final ErrorCallback errorCallback, Charset charset) {
    if (done) {
        throw UndertowMessages.MESSAGES.requestBodyAlreadyRead();
    }
    final ErrorCallback error = errorCallback == null ? END_EXCHANGE : errorCallback;
    if (callback == null) {
        throw UndertowMessages.MESSAGES.argumentCannotBeNull("callback");
    }
    if (exchange.isRequestComplete()) {
        callback.handle(exchange, "", true);
        return;
    }
    String contentLengthString = exchange.getRequestHeaders().getFirst(Headers.CONTENT_LENGTH);
    long contentLength;
    if (contentLengthString != null) {
        contentLength = Long.parseLong(contentLengthString);
        if (contentLength > Integer.MAX_VALUE) {
            error.error(exchange, new RequestToLargeException());
            return;
        }
    } else {
        contentLength = -1;
    }
    if (maxBufferSize > 0) {
        if (contentLength > maxBufferSize) {
            error.error(exchange, new RequestToLargeException());
            return;
        }
    }
    final CharsetDecoder decoder = charset.newDecoder();
    PooledByteBuffer pooled = exchange.getConnection().getByteBufferPool().allocate();
    final ByteBuffer buffer = pooled.getBuffer();
    channel.getReadSetter().set(new ChannelListener<StreamSourceChannel>() {

        @Override
        public void handleEvent(final StreamSourceChannel channel) {
            if (done || paused) {
                return;
            }
            PooledByteBuffer pooled = exchange.getConnection().getByteBufferPool().allocate();
            final ByteBuffer buffer = pooled.getBuffer();
            try {
                int res;
                do {
                    if (paused) {
                        return;
                    }
                    try {
                        buffer.clear();
                        res = channel.read(buffer);
                        if (res == -1) {
                            done = true;
                            Connectors.executeRootHandler(new HttpHandler() {

                                @Override
                                public void handleRequest(HttpServerExchange exchange) throws Exception {
                                    callback.handle(exchange, "", true);
                                }
                            }, exchange);
                            return;
                        } else if (res == 0) {
                            return;
                        } else {
                            buffer.flip();
                            final CharBuffer cb = decoder.decode(buffer);
                            Connectors.executeRootHandler(new HttpHandler() {

                                @Override
                                public void handleRequest(HttpServerExchange exchange) throws Exception {
                                    callback.handle(exchange, cb.toString(), false);
                                    if (!paused) {
                                        channel.resumeReads();
                                    } else {
                                        System.out.println("paused");
                                    }
                                }
                            }, exchange);
                        }
                    } catch (final IOException e) {
                        Connectors.executeRootHandler(new HttpHandler() {

                            @Override
                            public void handleRequest(HttpServerExchange exchange) throws Exception {
                                error.error(exchange, e);
                            }
                        }, exchange);
                        return;
                    }
                } while (true);
            } finally {
                pooled.close();
            }
        }
    });
    try {
        int res;
        do {
            try {
                buffer.clear();
                res = channel.read(buffer);
                if (res == -1) {
                    done = true;
                    callback.handle(exchange, "", true);
                    return;
                } else if (res == 0) {
                    channel.resumeReads();
                    return;
                } else {
                    buffer.flip();
                    CharBuffer cb = decoder.decode(buffer);
                    callback.handle(exchange, cb.toString(), false);
                    if (paused) {
                        return;
                    }
                }
            } catch (IOException e) {
                error.error(exchange, e);
                return;
            }
        } while (true);
    } finally {
        pooled.close();
    }
}

Example 33 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project undertow by undertow-io.

the class BlockingReceiverImpl method receivePartialString.

@Override
public void receivePartialString(final PartialStringCallback callback, final ErrorCallback errorCallback, Charset charset) {
    if (done) {
        throw UndertowMessages.MESSAGES.requestBodyAlreadyRead();
    }
    final ErrorCallback error = errorCallback == null ? END_EXCHANGE : errorCallback;
    if (callback == null) {
        throw UndertowMessages.MESSAGES.argumentCannotBeNull("callback");
    }
    if (exchange.isRequestComplete()) {
        callback.handle(exchange, "", true);
        return;
    }
    String contentLengthString = exchange.getRequestHeaders().getFirst(Headers.CONTENT_LENGTH);
    long contentLength;
    if (contentLengthString != null) {
        contentLength = Long.parseLong(contentLengthString);
        if (contentLength > Integer.MAX_VALUE) {
            error.error(exchange, new RequestToLargeException());
            return;
        }
    } else {
        contentLength = -1;
    }
    if (maxBufferSize > 0) {
        if (contentLength > maxBufferSize) {
            error.error(exchange, new RequestToLargeException());
            return;
        }
    }
    CharsetDecoder decoder = charset.newDecoder();
    int s;
    try (PooledByteBuffer pooled = exchange.getConnection().getByteBufferPool().getArrayBackedPool().allocate()) {
        while ((s = inputStream.read(pooled.getBuffer().array(), pooled.getBuffer().arrayOffset(), pooled.getBuffer().remaining())) > 0) {
            pooled.getBuffer().limit(s);
            CharBuffer res = decoder.decode(pooled.getBuffer());
            callback.handle(exchange, res.toString(), false);
            pooled.getBuffer().clear();
        }
        callback.handle(exchange, "", true);
    } catch (IOException e) {
        error.error(exchange, e);
    }
}

Example 34 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project jdk8u_jdk by JetBrains.

the class StringCoding method decode.

static char[] decode(Charset cs, byte[] ba, int off, int len) {
    // (1)We never cache the "external" cs, the only benefit of creating
    // an additional StringDe/Encoder object to wrap it is to share the
    // de/encode() method. These SD/E objects are short-lifed, the young-gen
    // gc should be able to take care of them well. But the best approash
    // is still not to generate them if not really necessary.
    // (2)The defensive copy of the input byte/char[] has a big performance
    // impact, as well as the outgoing result byte/char[]. Need to do the
    // optimization check of (sm==null && classLoader0==null) for both.
    // (3)getClass().getClassLoader0() is expensive
    // (4)There might be a timing gap in isTrusted setting. getClassLoader0()
    // is only chcked (and then isTrusted gets set) when (SM==null). It is
    // possible that the SM==null for now but then SM is NOT null later
    // when safeTrim() is invoked...the "safe" way to do is to redundant
    // check (... && (isTrusted || SM == null || getClassLoader0())) in trim
    // but it then can be argued that the SM is null when the opertaion
    // is started...
    CharsetDecoder cd = cs.newDecoder();
    int en = scale(len, cd.maxCharsPerByte());
    char[] ca = new char[en];
    if (len == 0)
        return ca;
    boolean isTrusted = false;
    if (System.getSecurityManager() != null) {
        if (!(isTrusted = (cs.getClass().getClassLoader0() == null))) {
            ba = Arrays.copyOfRange(ba, off, off + len);
            off = 0;
        }
    }
    cd.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE).reset();
    if (cd instanceof ArrayDecoder) {
        int clen = ((ArrayDecoder) cd).decode(ba, off, len, ca);
        return safeTrim(ca, clen, cs, isTrusted);
    } else {
        ByteBuffer bb = ByteBuffer.wrap(ba, off, len);
        CharBuffer cb = CharBuffer.wrap(ca);
        try {
            CoderResult cr = cd.decode(bb, cb, true);
            if (!cr.isUnderflow())
                cr.throwException();
            cr = cd.flush(cb);
            if (!cr.isUnderflow())
                cr.throwException();
        } catch (CharacterCodingException x) {
            // so this shouldn't happen
            throw new Error(x);
        }
        return safeTrim(ca, cb.position(), cs, isTrusted);
    }
}

Example 35 with CharsetDecoder

use of java.nio.charset.CharsetDecoder in project jdk8u_jdk by JetBrains.

the class ZipCoder method toString.

String toString(byte[] ba, int length) {
    CharsetDecoder cd = decoder().reset();
    int len = (int) (length * cd.maxCharsPerByte());
    char[] ca = new char[len];
    if (len == 0)
        return new String(ca);
    // REPORT mode.
    if (isUTF8 && cd instanceof ArrayDecoder) {
        int clen = ((ArrayDecoder) cd).decode(ba, 0, length, ca);
        if (// malformed
        clen == -1)
            throw new IllegalArgumentException("MALFORMED");
        return new String(ca, 0, clen);
    }
    ByteBuffer bb = ByteBuffer.wrap(ba, 0, length);
    CharBuffer cb = CharBuffer.wrap(ca);
    CoderResult cr = cd.decode(bb, cb, true);
    if (!cr.isUnderflow())
        throw new IllegalArgumentException(cr.toString());
    cr = cd.flush(cb);
    if (!cr.isUnderflow())
        throw new IllegalArgumentException(cr.toString());
    return new String(ca, 0, cb.position());
}