Examples with SchemaBuilder org.apache.kafka.connect.data.SchemaBuilder used on opensource projects

Example 86 with SchemaBuilder

use of org.apache.kafka.connect.data.SchemaBuilder in project debezium by debezium.

the class ByLogicalTableRouter method updateKeySchema.

private Schema updateKeySchema(Schema oldKeySchema, String newTopicName) {
    Schema newKeySchema = keySchemaUpdateCache.get(oldKeySchema);
    if (newKeySchema != null) {
        return newKeySchema;
    }
    final SchemaBuilder builder = copySchemaExcludingName(oldKeySchema, SchemaBuilder.struct());
    builder.name(schemaNameAdjuster.adjust(newTopicName + ".Key"));
    // Now that multiple physical tables can share a topic, the event's key may need to be augmented to include
    // fields other than just those for the record's primary/unique key, since these are not guaranteed to be unique
    // across tables. We need some identifier added to the key that distinguishes the different physical tables.
    builder.field(keyFieldName, Schema.STRING_SCHEMA);
    newKeySchema = builder.build();
    keySchemaUpdateCache.put(oldKeySchema, newKeySchema);
    return newKeySchema;
}

Example 87 with SchemaBuilder

use of org.apache.kafka.connect.data.SchemaBuilder in project debezium by debezium.

the class ByLogicalTableRouter method updateEnvelopeSchema.

private Schema updateEnvelopeSchema(Schema oldEnvelopeSchema, String newTopicName) {
    Schema newEnvelopeSchema = envelopeSchemaUpdateCache.get(oldEnvelopeSchema);
    if (newEnvelopeSchema != null) {
        return newEnvelopeSchema;
    }
    final Schema oldValueSchema = oldEnvelopeSchema.field(Envelope.FieldName.BEFORE).schema();
    final SchemaBuilder valueBuilder = copySchemaExcludingName(oldValueSchema, SchemaBuilder.struct());
    valueBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Value"));
    final Schema newValueSchema = valueBuilder.build();
    final SchemaBuilder envelopeBuilder = copySchemaExcludingName(oldEnvelopeSchema, SchemaBuilder.struct(), false);
    for (org.apache.kafka.connect.data.Field field : oldEnvelopeSchema.fields()) {
        final String fieldName = field.name();
        Schema fieldSchema = field.schema();
        if (Objects.equals(fieldName, Envelope.FieldName.BEFORE) || Objects.equals(fieldName, Envelope.FieldName.AFTER)) {
            fieldSchema = newValueSchema;
        }
        envelopeBuilder.field(fieldName, fieldSchema);
    }
    envelopeBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Envelope"));
    newEnvelopeSchema = envelopeBuilder.build();
    envelopeSchemaUpdateCache.put(oldEnvelopeSchema, newEnvelopeSchema);
    return newEnvelopeSchema;
}

Example 88 with SchemaBuilder

use of org.apache.kafka.connect.data.SchemaBuilder in project debezium by debezium.

the class TableSchemaBuilder method create.

/**
 * Create a {@link TableSchema} from the given {@link Table table definition}. The resulting TableSchema will have a
 * {@link TableSchema#keySchema() key schema} that contains all of the columns that make up the table's primary key,
 * and a {@link TableSchema#valueSchema() value schema} that contains only those columns that are not in the table's primary
 * key.
 * <p>
 * This is equivalent to calling {@code create(table,false)}.
 *
 * @param schemaPrefix the prefix added to the table identifier to construct the schema names; may be null if there is no
 *            prefix
 * @param envelopSchemaName the name of the schema of the built table's envelope
 * @param table the table definition; may not be null
 * @param filter the filter that specifies whether columns in the table should be included; may be null if all columns
 *            are to be included
 * @param mappers the mapping functions for columns; may be null if none of the columns are to be mapped to different values
 * @return the table schema that can be used for sending rows of data for this table to Kafka Connect; never null
 */
public TableSchema create(String schemaPrefix, String envelopSchemaName, Table table, Predicate<ColumnId> filter, ColumnMappers mappers) {
    if (schemaPrefix == null)
        schemaPrefix = "";
    // Build the schemas ...
    final TableId tableId = table.id();
    final String tableIdStr = tableId.toString();
    final String schemaNamePrefix = schemaPrefix + tableIdStr;
    LOGGER.debug("Mapping table '{}' to schemas under '{}'", tableId, schemaNamePrefix);
    SchemaBuilder valSchemaBuilder = SchemaBuilder.struct().name(schemaNameAdjuster.adjust(schemaNamePrefix + ".Value"));
    SchemaBuilder keySchemaBuilder = SchemaBuilder.struct().name(schemaNameAdjuster.adjust(schemaNamePrefix + ".Key"));
    AtomicBoolean hasPrimaryKey = new AtomicBoolean(false);
    table.columns().forEach(column -> {
        if (table.isPrimaryKeyColumn(column.name())) {
            // The column is part of the primary key, so ALWAYS add it to the PK schema ...
            addField(keySchemaBuilder, column, null);
            hasPrimaryKey.set(true);
        }
        if (filter == null || filter.test(new ColumnId(tableId, column.name()))) {
            // Add the column to the value schema only if the column has not been filtered ...
            ColumnMapper mapper = mappers == null ? null : mappers.mapperFor(tableId, column);
            addField(valSchemaBuilder, column, mapper);
        }
    });
    Schema valSchema = valSchemaBuilder.optional().build();
    Schema keySchema = hasPrimaryKey.get() ? keySchemaBuilder.build() : null;
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Mapped primary key for table '{}' to schema: {}", tableId, SchemaUtil.asDetailedString(keySchema));
        LOGGER.debug("Mapped columns for table '{}' to schema: {}", tableId, SchemaUtil.asDetailedString(valSchema));
    }
    Envelope envelope = Envelope.defineSchema().withName(schemaNameAdjuster.adjust(envelopSchemaName)).withRecord(valSchema).withSource(sourceInfoSchema).build();
    // Create the generators ...
    Function<Object[], Object> keyGenerator = createKeyGenerator(keySchema, tableId, table.primaryKeyColumns());
    Function<Object[], Struct> valueGenerator = createValueGenerator(valSchema, tableId, table.columns(), filter, mappers);
    // And the table schema ...
    return new TableSchema(keySchema, keyGenerator, envelope, valSchema, valueGenerator);
}