Examples with Marker io.debezium.text.TokenStream.Marker used on opensource projects

Example 6 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class MySqlDdlParser method parseCreateDefinition.

/**
 * @param isAlterStatement whether this is an ALTER TABLE statement or not (i.e. CREATE TABLE)
 */
protected void parseCreateDefinition(Marker start, TableEditor table, boolean isAlterStatement) {
    // If the first token is a quoted identifier, then we know it is a column name ...
    Collection<ParsingException> errors = null;
    boolean quoted = isNextTokenQuotedIdentifier();
    Marker defnStart = tokens.mark();
    if (!quoted) {
        // The first token is not quoted so let's check for other expressions ...
        if (tokens.canConsume("CHECK")) {
            // Try to parse the constraints first ...
            consumeExpression(start);
            return;
        }
        if (tokens.canConsume("CONSTRAINT", TokenStream.ANY_VALUE, "PRIMARY", "KEY") || tokens.canConsume("CONSTRAINT", "PRIMARY", "KEY") || tokens.canConsume("PRIMARY", "KEY")) {
            try {
                if (tokens.canConsume("USING")) {
                    parseIndexType(start);
                }
                if (!tokens.matches('(')) {
                    // index name
                    tokens.consume();
                }
                List<String> pkColumnNames = parseIndexColumnNames(start);
                table.setPrimaryKeyNames(pkColumnNames);
                parseIndexOptions(start);
                // MySQL does not allow a primary key to have nullable columns, so let's make sure we model that correctly ...
                pkColumnNames.forEach(name -> {
                    Column c = table.columnWithName(name);
                    if (c != null && c.isOptional()) {
                        table.addColumn(c.edit().optional(false).create());
                    }
                });
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsume("CONSTRAINT", TokenStream.ANY_VALUE, "UNIQUE") || tokens.canConsume("CONSTRAINT", "UNIQUE") || tokens.canConsume("UNIQUE")) {
            tokens.canConsumeAnyOf("KEY", "INDEX");
            try {
                if (!tokens.matches('(')) {
                    if (!tokens.matches("USING")) {
                        // name of unique index ...
                        tokens.consume();
                    }
                    if (tokens.matches("USING")) {
                        parseIndexType(start);
                    }
                }
                List<String> uniqueKeyColumnNames = parseIndexColumnNames(start);
                if (table.primaryKeyColumnNames().isEmpty()) {
                    // this may eventually get overwritten by a real PK
                    table.setPrimaryKeyNames(uniqueKeyColumnNames);
                }
                parseIndexOptions(start);
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsume("CONSTRAINT", TokenStream.ANY_VALUE, "FOREIGN", "KEY") || tokens.canConsume("FOREIGN", "KEY")) {
            try {
                if (!tokens.matches('(')) {
                    // name of foreign key
                    tokens.consume();
                }
                parseIndexColumnNames(start);
                if (tokens.matches("REFERENCES")) {
                    parseReferenceDefinition(start);
                }
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsumeAnyOf("INDEX", "KEY")) {
            try {
                if (!tokens.matches('(')) {
                    if (!tokens.matches("USING")) {
                        // name of unique index ...
                        tokens.consume();
                    }
                    if (tokens.matches("USING")) {
                        parseIndexType(start);
                    }
                }
                parseIndexColumnNames(start);
                parseIndexOptions(start);
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsumeAnyOf("FULLTEXT", "SPATIAL")) {
            try {
                tokens.canConsumeAnyOf("INDEX", "KEY");
                if (!tokens.matches('(')) {
                    // name of unique index ...
                    tokens.consume();
                }
                parseIndexColumnNames(start);
                parseIndexOptions(start);
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
    }
    try {
        // It's either quoted (meaning it's a column definition)
        if (isAlterStatement && !quoted) {
            // optional for ALTER TABLE
            tokens.canConsume("COLUMN");
        }
        String columnName = parseColumnName();
        parseCreateColumn(start, table, columnName, null);
    } catch (ParsingException e) {
        if (errors != null) {
            errors = accumulateParsingFailure(e, errors);
            throw new MultipleParsingExceptions(errors);
        }
        throw e;
    } catch (MultipleParsingExceptions e) {
        if (errors != null) {
            errors = accumulateParsingFailure(e, errors);
            throw new MultipleParsingExceptions(errors);
        }
        throw e;
    }
}

Example 7 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class MySqlDdlParser method parseVariableValue.

protected String parseVariableValue() {
    if (tokens.canConsumeAnyOf(",", ";")) {
        // The variable is blank ...
        return "";
    }
    Marker start = tokens.mark();
    tokens.consumeUntilEndOrOneOf(",", ";");
    String value = tokens.getContentFrom(start);
    if (value.startsWith("'") && value.endsWith("'")) {
        // Remove the single quotes around the value ...
        if (value.length() <= 2) {
            value = "";
        } else {
            value = value.substring(1, value.length() - 2);
        }
    }
    return value;
}

Example 8 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class MySqlDdlParser method sequentially.

/**
 * Try calling the supplied functions in sequence, stopping as soon as one of them succeeds.
 *
 * @param functions the functions
 */
@SuppressWarnings("unchecked")
protected void sequentially(Consumer<Marker>... functions) {
    if (functions == null || functions.length == 0)
        return;
    Collection<ParsingException> errors = new ArrayList<>();
    Marker marker = tokens.mark();
    for (Consumer<Marker> function : functions) {
        try {
            function.accept(marker);
            return;
        } catch (ParsingException e) {
            errors.add(e);
            tokens.rewind(marker);
        }
    }
    parsingFailed(marker.position(), errors, "One or more errors trying to parse statement");
}

Example 9 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class MySqlDdlParser method parseColumnDefinition.

protected void parseColumnDefinition(Marker start, String columnName, TokenStream tokens, TableEditor table, ColumnEditor column, AtomicBoolean isPrimaryKey) {
    // Parse the data type, which must be at this location ...
    List<ParsingException> errors = new ArrayList<>();
    Marker dataTypeStart = tokens.mark();
    DataType dataType = dataTypeParser.parse(tokens, errors::addAll);
    if (dataType == null) {
        String dataTypeName = parseDomainName(start);
        if (dataTypeName != null)
            dataType = DataType.userDefinedType(dataTypeName);
    }
    if (dataType == null) {
        // No data type was found
        parsingFailed(dataTypeStart.position(), errors, "Unable to read the data type");
        return;
    }
    column.jdbcType(dataType.jdbcType());
    column.type(dataType.name(), dataType.expression());
    if ("ENUM".equals(dataType.name())) {
        column.length(1);
    } else if ("SET".equals(dataType.name())) {
        List<String> options = parseSetAndEnumOptions(dataType.expression());
        // After DBZ-132, it will always be comma seperated
        // number of options + number of commas
        column.length(Math.max(0, options.size() * 2 - 1));
    } else {
        if (dataType.length() > -1)
            column.length((int) dataType.length());
        if (dataType.scale() > -1)
            column.scale(dataType.scale());
    }
    if (Types.NCHAR == dataType.jdbcType() || Types.NVARCHAR == dataType.jdbcType()) {
        // NCHAR and NVARCHAR columns always uses utf8 as charset
        column.charsetName("utf8");
    }
    if (Types.DECIMAL == dataType.jdbcType()) {
        if (dataType.length() == -1) {
            column.length(10);
        }
        if (dataType.scale() == -1) {
            column.scale(0);
        }
    }
    if (tokens.canConsume("CHARSET") || tokens.canConsume("CHARACTER", "SET")) {
        String charsetName = tokens.consume();
        if (!"DEFAULT".equalsIgnoreCase(charsetName)) {
            // Only record it if not inheriting the character set from the table
            column.charsetName(charsetName);
        }
    }
    if (tokens.canConsume("COLLATE")) {
        // name of collation
        tokens.consume();
    }
    if (tokens.canConsume("AS") || tokens.canConsume("GENERATED", "ALWAYS", "AS")) {
        consumeExpression(start);
        tokens.canConsumeAnyOf("VIRTUAL", "STORED");
        if (tokens.canConsume("UNIQUE")) {
            tokens.canConsume("KEY");
        }
        if (tokens.canConsume("COMMENT")) {
            consumeQuotedString();
        }
        tokens.canConsume("NOT", "NULL");
        tokens.canConsume("NULL");
        tokens.canConsume("PRIMARY", "KEY");
        tokens.canConsume("KEY");
    } else {
        while (tokens.matchesAnyOf("NOT", "NULL", "DEFAULT", "AUTO_INCREMENT", "UNIQUE", "PRIMARY", "KEY", "COMMENT", "REFERENCES", "COLUMN_FORMAT", "ON", "COLLATE")) {
            // Nullability ...
            if (tokens.canConsume("NOT", "NULL")) {
                column.optional(false);
            } else if (tokens.canConsume("NULL")) {
                column.optional(true);
            }
            // Default value ...
            if (tokens.matches("DEFAULT")) {
                parseDefaultClause(start);
            }
            if (tokens.matches("ON", "UPDATE") || tokens.matches("ON", "DELETE")) {
                parseOnUpdateOrDelete(tokens.mark());
                column.autoIncremented(true);
            }
            // Other options ...
            if (tokens.canConsume("AUTO_INCREMENT")) {
                column.autoIncremented(true);
                column.generated(true);
            }
            if (tokens.canConsume("UNIQUE", "KEY") || tokens.canConsume("UNIQUE")) {
                if (table.primaryKeyColumnNames().isEmpty() && !column.isOptional()) {
                    // The table has no primary key (yet) but this is a non-null column and therefore will have all unique
                    // values (MySQL allows UNIQUE indexes with some nullable columns, but in that case allows duplicate
                    // rows),
                    // so go ahead and set it to this column as it's a unique key
                    isPrimaryKey.set(true);
                }
            }
            if (tokens.canConsume("PRIMARY", "KEY") || tokens.canConsume("KEY")) {
                // Always set this column as the primary key
                // MySQL primary key columns may not be null
                column.optional(false);
                isPrimaryKey.set(true);
            }
            if (tokens.canConsume("COMMENT")) {
                consumeQuotedString();
            }
            if (tokens.canConsume("COLUMN_FORMAT")) {
                tokens.consumeAnyOf("FIXED", "DYNAMIC", "DEFAULT");
            }
            if (tokens.matches("REFERENCES")) {
                parseReferenceDefinition(start);
            }
            if (tokens.canConsume("COLLATE")) {
                // name of collation
                tokens.consume();
            }
        }
    }
}

Example 10 with Marker

use of io.debezium.text.TokenStream.Marker in project debezium by debezium.

the class DdlParser method parse.

/**
 * Examine the stream starting at its current position for DDL statements, and apply those statements to the specified
 * database table definitions.
 *
 * @param ddlContent the stream of tokens containing the DDL statements; may not be null
 * @param databaseTables the database's table definitions, which should be used by this method to create, change, or remove
 *            tables as defined in the DDL content; may not be null
 * @throws ParsingException if there is a problem parsing the supplied content
 * @throws IllegalStateException if the supplied token stream is in an invalid state
 */
public final void parse(TokenStream ddlContent, Tables databaseTables) throws ParsingException, IllegalStateException {
    this.tokens = ddlContent;
    this.databaseTables = databaseTables;
    Marker marker = ddlContent.mark();
    try {
        while (ddlContent.hasNext()) {
            parseNextStatement(ddlContent.mark());
            // Consume the statement terminator if it is still there ...
            tokens.canConsume(DdlTokenizer.STATEMENT_TERMINATOR);
        }
    } catch (ParsingException e) {
        ddlContent.rewind(marker);
        throw new ParsingException(e.getPosition(), "Failed to parse statement '" + ddlContent.getInputString() + "'", e);
    } catch (Throwable t) {
        parsingFailed(ddlContent.nextPosition(), "Unexpected exception while parsing statement " + ddlContent.getInputString(), t);
    }
}