Search in sources :

Example 1 with LookupJoinWithCalcRunner

use of org.apache.flink.table.runtime.operators.join.lookup.LookupJoinWithCalcRunner in project flink by apache.

the class LookupJoinHarnessTest method createHarness.

// ---------------------------------------------------------------------------------
@SuppressWarnings("unchecked")
private OneInputStreamOperatorTestHarness<RowData, RowData> createHarness(JoinType joinType, FilterOnTable filterOnTable) throws Exception {
    boolean isLeftJoin = joinType == JoinType.LEFT_JOIN;
    ProcessFunction<RowData, RowData> joinRunner;
    if (filterOnTable == FilterOnTable.WITHOUT_FILTER) {
        joinRunner = new LookupJoinRunner(new GeneratedFunctionWrapper<>(new TestingFetcherFunction()), new GeneratedCollectorWrapper<>(new TestingFetcherCollector()), isLeftJoin, 2);
    } else {
        joinRunner = new LookupJoinWithCalcRunner(new GeneratedFunctionWrapper<>(new TestingFetcherFunction()), new GeneratedFunctionWrapper<>(new CalculateOnTemporalTable()), new GeneratedCollectorWrapper<>(new TestingFetcherCollector()), isLeftJoin, 2);
    }
    ProcessOperator<RowData, RowData> operator = new ProcessOperator<>(joinRunner);
    return new OneInputStreamOperatorTestHarness<>(operator, inSerializer);
}
Also used : ProcessOperator(org.apache.flink.streaming.api.operators.ProcessOperator) GeneratedFunctionWrapper(org.apache.flink.table.runtime.generated.GeneratedFunctionWrapper) GeneratedCollectorWrapper(org.apache.flink.table.runtime.generated.GeneratedCollectorWrapper) LookupJoinRunner(org.apache.flink.table.runtime.operators.join.lookup.LookupJoinRunner) OneInputStreamOperatorTestHarness(org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness) GenericRowData(org.apache.flink.table.data.GenericRowData) RowData(org.apache.flink.table.data.RowData) JoinedRowData(org.apache.flink.table.data.utils.JoinedRowData) LookupJoinWithCalcRunner(org.apache.flink.table.runtime.operators.join.lookup.LookupJoinWithCalcRunner)

Example 2 with LookupJoinWithCalcRunner

use of org.apache.flink.table.runtime.operators.join.lookup.LookupJoinWithCalcRunner in project flink by apache.

the class CommonExecLookupJoin method createSyncLookupJoin.

private StreamOperatorFactory<RowData> createSyncLookupJoin(RelOptTable temporalTable, ExecNodeConfig config, Map<Integer, LookupJoinUtil.LookupKey> allLookupKeys, TableFunction<?> syncLookupFunction, RelBuilder relBuilder, RowType inputRowType, RowType tableSourceRowType, RowType resultRowType, boolean isLeftOuterJoin, boolean isObjectReuseEnabled) {
    DataTypeFactory dataTypeFactory = ShortcutUtils.unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
    int[] orderedLookupKeys = LookupJoinUtil.getOrderedLookupKeys(allLookupKeys.keySet());
    GeneratedFunction<FlatMapFunction<RowData, RowData>> generatedFetcher = LookupJoinCodeGenerator.generateSyncLookupFunction(config.getTableConfig(), dataTypeFactory, inputRowType, tableSourceRowType, resultRowType, allLookupKeys, orderedLookupKeys, syncLookupFunction, StringUtils.join(temporalTable.getQualifiedName(), "."), isObjectReuseEnabled);
    RowType rightRowType = Optional.ofNullable(temporalTableOutputType).map(FlinkTypeFactory::toLogicalRowType).orElse(tableSourceRowType);
    CodeGeneratorContext ctx = new CodeGeneratorContext(config.getTableConfig());
    GeneratedCollector<TableFunctionCollector<RowData>> generatedCollector = LookupJoinCodeGenerator.generateCollector(ctx, inputRowType, rightRowType, resultRowType, JavaScalaConversionUtil.toScala(Optional.ofNullable(joinCondition)), JavaScalaConversionUtil.toScala(Optional.empty()), true);
    ProcessFunction<RowData, RowData> processFunc;
    if (existCalcOnTemporalTable) {
        // a projection or filter after table source scan
        GeneratedFunction<FlatMapFunction<RowData, RowData>> generatedCalc = LookupJoinCodeGenerator.generateCalcMapFunction(config.getTableConfig(), JavaScalaConversionUtil.toScala(projectionOnTemporalTable), filterOnTemporalTable, temporalTableOutputType, tableSourceRowType);
        processFunc = new LookupJoinWithCalcRunner(generatedFetcher, generatedCalc, generatedCollector, isLeftOuterJoin, rightRowType.getFieldCount());
    } else {
        // right type is the same as table source row type, because no calc after temporal table
        processFunc = new LookupJoinRunner(generatedFetcher, generatedCollector, isLeftOuterJoin, rightRowType.getFieldCount());
    }
    return SimpleOperatorFactory.of(new ProcessOperator<>(processFunc));
}
Also used : CodeGeneratorContext(org.apache.flink.table.planner.codegen.CodeGeneratorContext) RowType(org.apache.flink.table.types.logical.RowType) DataTypeFactory(org.apache.flink.table.catalog.DataTypeFactory) TableFunctionCollector(org.apache.flink.table.runtime.collector.TableFunctionCollector) AsyncLookupJoinRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner) LookupJoinRunner(org.apache.flink.table.runtime.operators.join.lookup.LookupJoinRunner) RowData(org.apache.flink.table.data.RowData) FlatMapFunction(org.apache.flink.api.common.functions.FlatMapFunction) LookupJoinWithCalcRunner(org.apache.flink.table.runtime.operators.join.lookup.LookupJoinWithCalcRunner) AsyncLookupJoinWithCalcRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinWithCalcRunner)

Aggregations

RowData (org.apache.flink.table.data.RowData)2 LookupJoinRunner (org.apache.flink.table.runtime.operators.join.lookup.LookupJoinRunner)2 LookupJoinWithCalcRunner (org.apache.flink.table.runtime.operators.join.lookup.LookupJoinWithCalcRunner)2 FlatMapFunction (org.apache.flink.api.common.functions.FlatMapFunction)1 ProcessOperator (org.apache.flink.streaming.api.operators.ProcessOperator)1 OneInputStreamOperatorTestHarness (org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness)1 DataTypeFactory (org.apache.flink.table.catalog.DataTypeFactory)1 GenericRowData (org.apache.flink.table.data.GenericRowData)1 JoinedRowData (org.apache.flink.table.data.utils.JoinedRowData)1 CodeGeneratorContext (org.apache.flink.table.planner.codegen.CodeGeneratorContext)1 TableFunctionCollector (org.apache.flink.table.runtime.collector.TableFunctionCollector)1 GeneratedCollectorWrapper (org.apache.flink.table.runtime.generated.GeneratedCollectorWrapper)1 GeneratedFunctionWrapper (org.apache.flink.table.runtime.generated.GeneratedFunctionWrapper)1 AsyncLookupJoinRunner (org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner)1 AsyncLookupJoinWithCalcRunner (org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinWithCalcRunner)1 RowType (org.apache.flink.table.types.logical.RowType)1