Examples with Tokenizer org.apache.hc.core5.util.Tokenizer used on opensource projects

Example 1 with Tokenizer

use of org.apache.hc.core5.util.Tokenizer in project httpcomponents-core by apache.

the class URLEncodedUtils method parse.

/**
 * Returns a list of {@link NameValuePair}s parameters.
 *
 * @param s input text.
 * @param charset parameter charset.
 * @param separators parameter separators.
 * @return list of query parameters.
 *
 * @since 4.4
 */
public static List<NameValuePair> parse(final CharSequence s, final Charset charset, final char... separators) {
    Args.notNull(s, "Char sequence");
    final Tokenizer tokenParser = Tokenizer.INSTANCE;
    final BitSet delimSet = new BitSet();
    for (final char separator : separators) {
        delimSet.set(separator);
    }
    final Tokenizer.Cursor cursor = new Tokenizer.Cursor(0, s.length());
    final List<NameValuePair> list = new ArrayList<>();
    while (!cursor.atEnd()) {
        delimSet.set('=');
        final String name = tokenParser.parseToken(s, cursor, delimSet);
        String value = null;
        if (!cursor.atEnd()) {
            final int delim = s.charAt(cursor.getPos());
            cursor.updatePos(cursor.getPos() + 1);
            if (delim == '=') {
                delimSet.clear('=');
                value = tokenParser.parseToken(s, cursor, delimSet);
                if (!cursor.atEnd()) {
                    cursor.updatePos(cursor.getPos() + 1);
                }
            }
        }
        if (!name.isEmpty()) {
            list.add(new BasicNameValuePair(PercentCodec.decode(name, charset, true), PercentCodec.decode(value, charset, true)));
        }
    }
    return list;
}

Example 2 with Tokenizer

use of org.apache.hc.core5.util.Tokenizer in project httpcomponents-core by apache.

the class URIAuthority method parse.

static URIAuthority parse(final CharSequence s, final Tokenizer.Cursor cursor) throws URISyntaxException {
    final Tokenizer tokenizer = Tokenizer.INSTANCE;
    String userInfo = null;
    final int initPos = cursor.getPos();
    final String token = tokenizer.parseContent(s, cursor, URISupport.HOST_SEPARATORS);
    if (!cursor.atEnd() && s.charAt(cursor.getPos()) == '@') {
        cursor.updatePos(cursor.getPos() + 1);
        if (!TextUtils.isBlank(token)) {
            userInfo = token;
        }
    } else {
        // Rewind
        cursor.updatePos(initPos);
    }
    final Host host = Host.parse(s, cursor);
    return new URIAuthority(userInfo, host);
}

Example 3 with Tokenizer

use of org.apache.hc.core5.util.Tokenizer in project httpcomponents-core by apache.

the class URIBuilder method parseQuery.

static List<NameValuePair> parseQuery(final CharSequence s, final Charset charset, final boolean plusAsBlank) {
    if (s == null) {
        return null;
    }
    final Tokenizer tokenParser = Tokenizer.INSTANCE;
    final ParserCursor cursor = new ParserCursor(0, s.length());
    final List<NameValuePair> list = new ArrayList<>();
    while (!cursor.atEnd()) {
        final String name = tokenParser.parseToken(s, cursor, QUERY_PARAM_SEPARATORS);
        String value = null;
        if (!cursor.atEnd()) {
            final int delim = s.charAt(cursor.getPos());
            cursor.updatePos(cursor.getPos() + 1);
            if (delim == PARAM_VALUE_SEPARATOR) {
                value = tokenParser.parseToken(s, cursor, QUERY_VALUE_SEPARATORS);
                if (!cursor.atEnd()) {
                    cursor.updatePos(cursor.getPos() + 1);
                }
            }
        }
        if (!name.isEmpty()) {
            list.add(new BasicNameValuePair(PercentCodec.decode(name, charset, plusAsBlank), PercentCodec.decode(value, charset, plusAsBlank)));
        }
    }
    return list;
}

Example 4 with Tokenizer

use of org.apache.hc.core5.util.Tokenizer in project httpcomponents-core by apache.

the class Host method parse.

static Host parse(final CharSequence s, final Tokenizer.Cursor cursor) throws URISyntaxException {
    final Tokenizer tokenizer = Tokenizer.INSTANCE;
    final String hostName;
    final boolean ipv6Brackets = !cursor.atEnd() && s.charAt(cursor.getPos()) == '[';
    if (ipv6Brackets) {
        cursor.updatePos(cursor.getPos() + 1);
        hostName = tokenizer.parseContent(s, cursor, URISupport.IPV6_HOST_TERMINATORS);
        if (cursor.atEnd() || !(s.charAt(cursor.getPos()) == ']')) {
            throw URISupport.createException(s, cursor, "Expected an IPv6 closing bracket ']'");
        }
        cursor.updatePos(cursor.getPos() + 1);
        if (!InetAddressUtils.isIPv6Address(hostName)) {
            throw URISupport.createException(s, cursor, "Expected an IPv6 address");
        }
    } else {
        hostName = tokenizer.parseContent(s, cursor, URISupport.PORT_SEPARATORS);
    }
    String portText = null;
    if (!cursor.atEnd() && s.charAt(cursor.getPos()) == ':') {
        cursor.updatePos(cursor.getPos() + 1);
        portText = tokenizer.parseContent(s, cursor, URISupport.TERMINATORS);
    }
    final int port;
    if (!TextUtils.isBlank(portText)) {
        if (!ipv6Brackets && portText.contains(":")) {
            throw URISupport.createException(s, cursor, "Expected IPv6 address to be enclosed in brackets");
        }
        try {
            port = Integer.parseInt(portText);
        } catch (final NumberFormatException ex) {
            throw URISupport.createException(s, cursor, "Port is invalid");
        }
    } else {
        port = -1;
    }
    return new Host(hostName, port);
}