Examples with TokenStream io.debezium.text.TokenStream used on opensource projects

Example 1 with TokenStream

use of io.debezium.text.TokenStream in project debezium by debezium.

the class DataTypeGrammarParser method parse.

/**
 * Parse the supplied grammar for a data type.
 *
 * @param jdbcType the {@link Types JDBC data type}
 * @param dataTypeDefn the data type grammar
 * @return the data type pattern that can be used for pattern matching the data type; never null
 * @throws ParsingException if the grammar cannot be parsed correctly
 */
public DataTypePattern parse(int jdbcType, String dataTypeDefn) throws ParsingException {
    TokenStream stream = new TokenStream(dataTypeDefn, tokenizer, false);
    stream.start();
    Pattern pattern = parseMultiple(stream);
    return pattern != null ? new DataTypePattern(pattern, jdbcType) : null;
}

Example 2 with TokenStream

use of io.debezium.text.TokenStream in project debezium by debezium.

the class TableIdParser method parse.

public static List<String> parse(String identifier) {
    TokenStream stream = new TokenStream(identifier, new TableIdTokenizer(identifier), true);
    stream.start();
    List<String> parts = new ArrayList<>();
    while (stream.hasNext()) {
        parts.add(stream.consume().replaceAll("''", "'").replaceAll("\"\"", "\"").replaceAll("``", "`"));
    }
    return parts;
}

Example 3 with TokenStream

use of io.debezium.text.TokenStream in project debezium by debezium.

the class MySqlDdlParser method parseSetAndEnumOptions.

/**
 * Parse the {@code ENUM} or {@code SET} data type expression to extract the character options, where the index(es) appearing
 * in the {@code ENUM} or {@code SET} values can be used to identify the acceptable characters.
 *
 * @param typeExpression the data type expression
 * @return the string containing the character options allowed by the {@code ENUM} or {@code SET}; never null
 */
public static List<String> parseSetAndEnumOptions(String typeExpression) {
    List<String> options = new ArrayList<>();
    TokenStream tokens = new TokenStream(typeExpression, TokenStream.basicTokenizer(false), false);
    tokens.start();
    if (tokens.canConsumeAnyOf("ENUM", "SET") && tokens.canConsume('(')) {
        // The literals should be quoted values ...
        if (tokens.matchesAnyOf(DdlTokenizer.DOUBLE_QUOTED_STRING, DdlTokenizer.SINGLE_QUOTED_STRING)) {
            options.add(withoutQuotes(tokens.consume()));
            while (tokens.canConsume(',')) {
                if (tokens.matchesAnyOf(DdlTokenizer.DOUBLE_QUOTED_STRING, DdlTokenizer.SINGLE_QUOTED_STRING)) {
                    options.add(withoutQuotes(tokens.consume()));
                }
            }
        }
        tokens.consume(')');
    }
    return options;
}

Example 4 with TokenStream

use of io.debezium.text.TokenStream in project debezium by debezium.

the class DdlParser method parse.

/**
 * Examine the supplied string containing DDL statements, and apply those statements to the specified
 * database table definitions.
 *
 * @param ddlContent the stream of tokens containing the DDL statements; may not be null
 * @param databaseTables the database's table definitions, which should be used by this method to create, change, or remove
 *            tables as defined in the DDL content; may not be null
 * @throws ParsingException if there is a problem parsing the supplied content
 */
public final void parse(String ddlContent, Tables databaseTables) {
    TokenStream stream = new TokenStream(ddlContent, new DdlTokenizer(!skipComments(), this::determineTokenType), false);
    stream.start();
    parse(stream, databaseTables);
}