Examples with Marker io.debezium.text.TokenStream.Marker used on opensource projects

Example 1 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class DataTypeParser method parse.

/**
 * Examine the stream starting at its current position for a matching data type. If this method finds a matching data type,
 * it will consume the stream of all tokens that make up the data type. However, if no data type is found, the stream is left
 * unchanged.
 * <p>
 * This method looks for data types that match those registered patterns.
 * This method also looks for multi-dimensional arrays, where any registered data type pattern is followed by one or more
 * array dimensions of the form {@code [n]}, where {@code n} is the integer dimension.
 * <p>
 * Sometimes, a data type matches one of the registered patterns but it contains a malformed length/precision, scale, and/or
 * array dimensions. These parsing exceptions can be reported back to the caller via the supplied {@code errorHandler},
 * although these errors are only reported when no data type is found. This is often useful when the caller expects
 * to find a data type, but no such data type can be found due to one or more parsing exceptions. When this happens,
 * the method calls {@code errorHandler} with all of the {@link ParsingException}s and then returns <code>null</code>.
 *
 * @param stream the stream of tokens; may not be null
 * @param errorHandler a function that should be called when no data type was found because at least one
 *            {@link ParsingException}
 *            occurred with the length/precision, scale, and/or array dimensions; may be null
 * @return the data type if one was found, or null if none were found
 */
public DataType parse(TokenStream stream, Consumer<Collection<ParsingException>> errorHandler) {
    if (stream.hasNext()) {
        // Look for all patterns that begin with the first token ...
        Collection<DataTypePattern> matchingPatterns = patterns.get(stream.peek().toUpperCase());
        if (matchingPatterns != null) {
            // At least one registered type begins with the first token, so go through them all in order ...
            ErrorCollector errors = new ErrorCollector();
            Marker mostReadMarker = null;
            DataType mostReadType = null;
            Marker marker = stream.mark();
            for (DataTypePattern pattern : matchingPatterns) {
                DataType result = pattern.match(stream, errors::record);
                if (result != null) {
                    // We found a match, so record it if it is better than our previous best ...
                    if (!stream.hasNext()) {
                        // There's no more to read, so we should be done ...
                        return result;
                    }
                    Marker endMarker = stream.mark();
                    if (mostReadMarker == null || endMarker.compareTo(mostReadMarker) > 0) {
                        mostReadMarker = endMarker;
                        mostReadType = result;
                    }
                }
                // always, even in the case of success
                stream.rewind(marker);
            }
            if (mostReadType != null) {
                // We've found at least one, so advance the stream to the end of what was consumed by that data type
                // and return the type that consumes the most of the stream ...
                stream.advance(mostReadMarker);
                return mostReadType;
            }
            // We still haven't found a data type ...
            errors.send(errorHandler);
        }
    }
    // Ultimately did not find a match ...
    return null;
}

Example 2 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class DdlParser method parseColumnsInSelectClause.

/**
 * Parse the column information in the SELECT clause. This statement stops before consuming the FROM clause.
 *
 * @param start the start of the statement
 * @return the map of resolved Columns keyed by the column alias (or name) used in the SELECT statement; never null but
 *         possibly
 *         empty if we couldn't parse the SELECT clause correctly
 */
protected Map<String, Column> parseColumnsInSelectClause(Marker start) {
    // Parse the column names ...
    Map<String, String> tableAliasByColumnAlias = new LinkedHashMap<>();
    Map<String, String> columnNameByAliases = new LinkedHashMap<>();
    parseColumnName(start, tableAliasByColumnAlias, columnNameByAliases);
    while (tokens.canConsume(',')) {
        parseColumnName(start, tableAliasByColumnAlias, columnNameByAliases);
    }
    // Parse the FROM clause, but we'll back up to the start of this before we return ...
    Marker startOfFrom = tokens.mark();
    Map<String, Column> columnsByName = new LinkedHashMap<>();
    Map<String, Table> fromTablesByAlias = parseSelectFromClause(start);
    Table singleTable = fromTablesByAlias.size() == 1 ? fromTablesByAlias.values().stream().findFirst().get() : null;
    tableAliasByColumnAlias.forEach((columnAlias, tableAlias) -> {
        // Resolve the alias into the actual column name in the referenced table ...
        String columnName = columnNameByAliases.getOrDefault(columnAlias, columnAlias);
        Column column = null;
        if (tableAlias == null) {
            // The column was not qualified with a table, so there should be a single table ...
            column = singleTable == null ? null : singleTable.columnWithName(columnName);
        } else {
            // The column was qualified with a table, so look it up ...
            Table table = fromTablesByAlias.get(tableAlias);
            column = table == null ? null : table.columnWithName(columnName);
        }
        if (column == null) {
            // Check to see whether the column name contains a constant value, in which case we need to create an
            // artificial column ...
            column = createColumnFromConstant(columnAlias, columnName);
        }
        // column may be null
        columnsByName.put(columnAlias, column);
    });
    tokens.rewind(startOfFrom);
    return columnsByName;
}

Example 3 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class DdlParser method consumeStatement.

/**
 * Consume all tokens from the current position that is a {@link #initializeStatementStarts(TokenSet) starting-statement
 * token} until either the
 * {@link #terminator() end-of-statement terminator token} or before the next
 * {@link #initializeStatementStarts(TokenSet) starting-statement token}.
 *
 * @throws ParsingException if the next token is not a {@link #initializeStatementStarts(TokenSet) starting-statement token}
 */
protected void consumeStatement() throws ParsingException {
    Marker start = tokens.mark();
    tokens.consume(DdlTokenizer.STATEMENT_KEY);
    consumeRemainingStatement(start);
}

Example 4 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class DdlParserSql2003 method parseColumnDefinition.

protected void parseColumnDefinition(Marker start, String columnName, TokenStream tokens, TableEditor table, ColumnEditor column, AtomicBoolean isPrimaryKey) {
    // Parse the data type, which must be at this location ...
    List<ParsingException> errors = new ArrayList<>();
    Marker dataTypeStart = tokens.mark();
    DataType dataType = dataTypeParser.parse(tokens, errors::addAll);
    if (dataType == null) {
        String dataTypeName = parseDomainName(start);
        if (dataTypeName != null)
            dataType = DataType.userDefinedType(dataTypeName);
    }
    if (dataType == null) {
        // No data type was found
        parsingFailed(dataTypeStart.position(), errors, "Unable to read the data type");
        return;
    }
    column.jdbcType(dataType.jdbcType());
    column.type(dataType.name(), dataType.expression());
    if (dataType.length() > -1)
        column.length((int) dataType.length());
    if (dataType.scale() > -1)
        column.scale(dataType.scale());
    if (tokens.matches("REFERENCES", "ARE")) {
        parseReferencesScopeCheck(start, columnName, tokens, column);
    }
    if (tokens.matches("DEFAULT")) {
        parseDefaultClause(start, column);
    } else if (tokens.matches("GENERATED")) {
        parseIdentityColumnSpec(start, column);
    }
    while (tokens.matchesAnyOf("NOT", "UNIQUE", "PRIMARY", "CHECK", "REFERENCES", "CONSTRAINT")) {
        parseColumnConstraintDefinition(start, column, isPrimaryKey);
    }
    if (tokens.canConsume("COLLATE")) {
        parseSchemaQualifiedName(start);
    }
}

Example 5 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class MySqlDdlParser method parseAlterTable.

protected void parseAlterTable(Marker start) {
    tokens.canConsume("IGNORE");
    tokens.consume("TABLE");
    TableId tableId = parseQualifiedTableName(start);
    TableEditor table = databaseTables.editTable(tableId);
    TableId oldTableId = null;
    if (table != null) {
        AtomicReference<TableId> newTableName = new AtomicReference<>(null);
        if (!tokens.matches(terminator()) && !tokens.matches("PARTITION")) {
            parseAlterSpecificationList(start, table, newTableName::set);
        }
        if (tokens.matches("PARTITION")) {
            parsePartitionOptions(start, table);
        }
        databaseTables.overwriteTable(table.create());
        if (newTableName.get() != null) {
            // the table was renamed ...
            Table renamed = databaseTables.renameTable(tableId, newTableName.get());
            if (renamed != null) {
                oldTableId = tableId;
                tableId = renamed.id();
            }
        }
    } else {
        Marker marker = tokens.mark();
        try {
            // We don't know about this table but we still have to parse the statement ...
            table = TableEditor.noOp(tableId);
            if (!tokens.matches(terminator()) && !tokens.matches("PARTITION")) {
                parseAlterSpecificationList(start, table, str -> {
                });
            }
            if (tokens.matches("PARTITION")) {
                parsePartitionOptions(start, table);
            }
            parseTableOptions(start, table);
        // do nothing with this
        } catch (ParsingException e) {
            tokens.rewind(marker);
            consumeRemainingStatement(start);
        }
    }
    signalAlterTable(tableId, oldTableId, start);
}