Examples with ChangelogMode org.apache.flink.table.connector.ChangelogMode used on opensource projects

Example 11 with ChangelogMode

use of org.apache.flink.table.connector.ChangelogMode in project flink by apache.

the class DynamicSourceUtils method isSourceChangeEventsDuplicate.

/**
 * Returns true if the table source produces duplicate change events.
 */
public static boolean isSourceChangeEventsDuplicate(ResolvedSchema resolvedSchema, DynamicTableSource tableSource, TableConfig tableConfig) {
    if (!(tableSource instanceof ScanTableSource)) {
        return false;
    }
    ChangelogMode mode = ((ScanTableSource) tableSource).getChangelogMode();
    boolean isCDCSource = !mode.containsOnly(RowKind.INSERT) && !isUpsertSource(resolvedSchema, tableSource);
    boolean changeEventsDuplicate = tableConfig.getConfiguration().getBoolean(ExecutionConfigOptions.TABLE_EXEC_SOURCE_CDC_EVENTS_DUPLICATE);
    boolean hasPrimaryKey = resolvedSchema.getPrimaryKey().isPresent();
    return isCDCSource && changeEventsDuplicate && hasPrimaryKey;
}

Example 12 with ChangelogMode

use of org.apache.flink.table.connector.ChangelogMode in project flink by apache.

the class TestValuesTableFactory method createDynamicTableSink.

@Override
public DynamicTableSink createDynamicTableSink(Context context) {
    FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
    helper.validate();
    String sinkClass = helper.getOptions().get(TABLE_SINK_CLASS);
    boolean isInsertOnly = helper.getOptions().get(SINK_INSERT_ONLY);
    String runtimeSink = helper.getOptions().get(RUNTIME_SINK);
    int expectedNum = helper.getOptions().get(SINK_EXPECTED_MESSAGES_NUM);
    Integer parallelism = helper.getOptions().get(SINK_PARALLELISM);
    boolean dropLateEvent = helper.getOptions().get(SINK_DROP_LATE_EVENT);
    final Map<String, DataType> writableMetadata = convertToMetadataMap(helper.getOptions().get(WRITABLE_METADATA), context.getClassLoader());
    final ChangelogMode changelogMode = Optional.ofNullable(helper.getOptions().get(SINK_CHANGELOG_MODE_ENFORCED)).map(m -> parseChangelogMode(m)).orElse(null);
    final DataType consumedType = context.getCatalogTable().getSchema().toPhysicalRowDataType();
    final int[] primaryKeyIndices = TableSchemaUtils.getPrimaryKeyIndices(context.getCatalogTable().getSchema());
    if (sinkClass.equals("DEFAULT")) {
        int rowTimeIndex = validateAndExtractRowtimeIndex(context.getCatalogTable(), dropLateEvent, isInsertOnly);
        return new TestValuesTableSink(consumedType, primaryKeyIndices, context.getObjectIdentifier().getObjectName(), isInsertOnly, runtimeSink, expectedNum, writableMetadata, parallelism, changelogMode, rowTimeIndex);
    } else {
        try {
            return InstantiationUtil.instantiate(sinkClass, DynamicTableSink.class, Thread.currentThread().getContextClassLoader());
        } catch (FlinkException e) {
            throw new TableException("Can't instantiate class " + sinkClass, e);
        }
    }
}

Example 13 with ChangelogMode

use of org.apache.flink.table.connector.ChangelogMode in project flink by apache.

the class TestValuesTableFactory method createDynamicTableSource.

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
    FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
    helper.validate();
    ChangelogMode changelogMode = parseChangelogMode(helper.getOptions().get(CHANGELOG_MODE));
    String runtimeSource = helper.getOptions().get(RUNTIME_SOURCE);
    boolean isBounded = helper.getOptions().get(BOUNDED);
    String dataId = helper.getOptions().get(DATA_ID);
    String sourceClass = helper.getOptions().get(TABLE_SOURCE_CLASS);
    boolean isAsync = helper.getOptions().get(ASYNC_ENABLED);
    String lookupFunctionClass = helper.getOptions().get(LOOKUP_FUNCTION_CLASS);
    boolean disableLookup = helper.getOptions().get(DISABLE_LOOKUP);
    boolean enableProjectionPushDown = helper.getOptions().get(ENABLE_PROJECTION_PUSH_DOWN);
    boolean nestedProjectionSupported = helper.getOptions().get(NESTED_PROJECTION_SUPPORTED);
    boolean enableWatermarkPushDown = helper.getOptions().get(ENABLE_WATERMARK_PUSH_DOWN);
    boolean failingSource = helper.getOptions().get(FAILING_SOURCE);
    int numElementToSkip = helper.getOptions().get(SOURCE_NUM_ELEMENT_TO_SKIP);
    boolean internalData = helper.getOptions().get(INTERNAL_DATA);
    Optional<List<String>> filterableFields = helper.getOptions().getOptional(FILTERABLE_FIELDS);
    Set<String> filterableFieldsSet = new HashSet<>();
    filterableFields.ifPresent(filterableFieldsSet::addAll);
    final Map<String, DataType> readableMetadata = convertToMetadataMap(helper.getOptions().get(READABLE_METADATA), context.getClassLoader());
    if (sourceClass.equals("DEFAULT")) {
        if (internalData) {
            return new TestValuesScanTableSourceWithInternalData(dataId, isBounded);
        }
        Collection<Row> data = registeredData.getOrDefault(dataId, Collections.emptyList());
        List<Map<String, String>> partitions = parsePartitionList(helper.getOptions().get(PARTITION_LIST));
        DataType producedDataType = context.getCatalogTable().getSchema().toPhysicalRowDataType();
        // pushing project into scan will prune schema and we have to get the mapping between
        // partition and row
        Map<Map<String, String>, Collection<Row>> partition2Rows;
        if (!partitions.isEmpty()) {
            partition2Rows = mapPartitionToRow(producedDataType, data, partitions);
        } else {
            // put all data into one partition
            partitions = Collections.emptyList();
            partition2Rows = new HashMap<>();
            partition2Rows.put(Collections.emptyMap(), data);
        }
        if (!enableProjectionPushDown) {
            return new TestValuesScanTableSourceWithoutProjectionPushDown(producedDataType, changelogMode, isBounded, runtimeSource, failingSource, partition2Rows, nestedProjectionSupported, null, Collections.emptyList(), filterableFieldsSet, numElementToSkip, Long.MAX_VALUE, partitions, readableMetadata, null);
        }
        if (disableLookup) {
            if (enableWatermarkPushDown) {
                return new TestValuesScanTableSourceWithWatermarkPushDown(producedDataType, changelogMode, runtimeSource, failingSource, partition2Rows, context.getObjectIdentifier().getObjectName(), nestedProjectionSupported, null, Collections.emptyList(), filterableFieldsSet, numElementToSkip, Long.MAX_VALUE, partitions, readableMetadata, null);
            } else {
                return new TestValuesScanTableSource(producedDataType, changelogMode, isBounded, runtimeSource, failingSource, partition2Rows, nestedProjectionSupported, null, Collections.emptyList(), filterableFieldsSet, numElementToSkip, Long.MAX_VALUE, partitions, readableMetadata, null);
            }
        } else {
            return new TestValuesScanLookupTableSource(producedDataType, changelogMode, isBounded, runtimeSource, failingSource, partition2Rows, isAsync, lookupFunctionClass, nestedProjectionSupported, null, Collections.emptyList(), filterableFieldsSet, numElementToSkip, Long.MAX_VALUE, partitions, readableMetadata, null);
        }
    } else {
        try {
            return InstantiationUtil.instantiate(sourceClass, DynamicTableSource.class, Thread.currentThread().getContextClassLoader());
        } catch (FlinkException e) {
            throw new TableException("Can't instantiate class " + sourceClass, e);
        }
    }
}