Examples with ProcTimeDeduplicateKeepLastRowFunction org.apache.flink.table.runtime.operators.deduplicate.ProcTimeDeduplicateKeepLastRowFunction used on opensource projects

Example 1 with ProcTimeDeduplicateKeepLastRowFunction

use of org.apache.flink.table.runtime.operators.deduplicate.ProcTimeDeduplicateKeepLastRowFunction in project flink by apache.

the class StreamExecChangelogNormalize method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final InternalTypeInfo<RowData> rowTypeInfo = (InternalTypeInfo<RowData>) inputTransform.getOutputType();
    final OneInputStreamOperator<RowData, RowData> operator;
    final long stateIdleTime = config.getStateRetentionTime();
    final boolean isMiniBatchEnabled = config.get(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED);
    GeneratedRecordEqualiser generatedEqualiser = new EqualiserCodeGenerator(rowTypeInfo.toRowType()).generateRecordEqualiser("DeduplicateRowEqualiser");
    if (isMiniBatchEnabled) {
        TypeSerializer<RowData> rowSerializer = rowTypeInfo.createSerializer(planner.getExecEnv().getConfig());
        ProcTimeMiniBatchDeduplicateKeepLastRowFunction processFunction = new ProcTimeMiniBatchDeduplicateKeepLastRowFunction(rowTypeInfo, rowSerializer, stateIdleTime, generateUpdateBefore, // generateInsert
        true, // inputInsertOnly
        false, generatedEqualiser);
        CountBundleTrigger<RowData> trigger = AggregateUtil.createMiniBatchTrigger(config);
        operator = new KeyedMapBundleOperator<>(processFunction, trigger);
    } else {
        ProcTimeDeduplicateKeepLastRowFunction processFunction = new ProcTimeDeduplicateKeepLastRowFunction(rowTypeInfo, stateIdleTime, generateUpdateBefore, // generateInsert
        true, // inputInsertOnly
        false, generatedEqualiser);
        operator = new KeyedProcessOperator<>(processFunction);
    }
    final OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(CHANGELOG_NORMALIZE_TRANSFORMATION, config), operator, rowTypeInfo, inputTransform.getParallelism());
    final RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(uniqueKeys, rowTypeInfo);
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    return transform;
}