Search in sources :

Example 1 with AsyncLookupJoinRunner

use of org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner in project flink by apache.

the class CommonExecLookupJoin method createAsyncLookupJoin.

@SuppressWarnings("unchecked")
private StreamOperatorFactory<RowData> createAsyncLookupJoin(RelOptTable temporalTable, ExecNodeConfig config, Map<Integer, LookupJoinUtil.LookupKey> allLookupKeys, AsyncTableFunction<Object> asyncLookupFunction, RelBuilder relBuilder, RowType inputRowType, RowType tableSourceRowType, RowType resultRowType, boolean isLeftOuterJoin) {
    int asyncBufferCapacity = config.get(ExecutionConfigOptions.TABLE_EXEC_ASYNC_LOOKUP_BUFFER_CAPACITY);
    long asyncTimeout = config.get(ExecutionConfigOptions.TABLE_EXEC_ASYNC_LOOKUP_TIMEOUT).toMillis();
    DataTypeFactory dataTypeFactory = ShortcutUtils.unwrapContext(relBuilder).getCatalogManager().getDataTypeFactory();
    LookupJoinCodeGenerator.GeneratedTableFunctionWithDataType<AsyncFunction<RowData, Object>> generatedFuncWithType = LookupJoinCodeGenerator.generateAsyncLookupFunction(config.getTableConfig(), dataTypeFactory, inputRowType, tableSourceRowType, resultRowType, allLookupKeys, LookupJoinUtil.getOrderedLookupKeys(allLookupKeys.keySet()), asyncLookupFunction, StringUtils.join(temporalTable.getQualifiedName(), "."));
    RowType rightRowType = Optional.ofNullable(temporalTableOutputType).map(FlinkTypeFactory::toLogicalRowType).orElse(tableSourceRowType);
    // a projection or filter after table source scan
    GeneratedResultFuture<TableFunctionResultFuture<RowData>> generatedResultFuture = LookupJoinCodeGenerator.generateTableAsyncCollector(config.getTableConfig(), "TableFunctionResultFuture", inputRowType, rightRowType, JavaScalaConversionUtil.toScala(Optional.ofNullable(joinCondition)));
    DataStructureConverter<?, ?> fetcherConverter = DataStructureConverters.getConverter(generatedFuncWithType.dataType());
    AsyncFunction<RowData, RowData> asyncFunc;
    if (existCalcOnTemporalTable) {
        // a projection or filter after table source scan
        GeneratedFunction<FlatMapFunction<RowData, RowData>> generatedCalc = LookupJoinCodeGenerator.generateCalcMapFunction(config.getTableConfig(), JavaScalaConversionUtil.toScala(projectionOnTemporalTable), filterOnTemporalTable, temporalTableOutputType, tableSourceRowType);
        asyncFunc = new AsyncLookupJoinWithCalcRunner(generatedFuncWithType.tableFunc(), (DataStructureConverter<RowData, Object>) fetcherConverter, generatedCalc, generatedResultFuture, InternalSerializers.create(rightRowType), isLeftOuterJoin, asyncBufferCapacity);
    } else {
        // right type is the same as table source row type, because no calc after temporal table
        asyncFunc = new AsyncLookupJoinRunner(generatedFuncWithType.tableFunc(), (DataStructureConverter<RowData, Object>) fetcherConverter, generatedResultFuture, InternalSerializers.create(rightRowType), isLeftOuterJoin, asyncBufferCapacity);
    }
    // when the downstream do not need orderness
    return new AsyncWaitOperatorFactory<>(asyncFunc, asyncTimeout, asyncBufferCapacity, AsyncDataStream.OutputMode.ORDERED);
}
Also used : AsyncLookupJoinRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner) AsyncWaitOperatorFactory(org.apache.flink.streaming.api.operators.async.AsyncWaitOperatorFactory) DataStructureConverter(org.apache.flink.table.data.conversion.DataStructureConverter) RowType(org.apache.flink.table.types.logical.RowType) DataTypeFactory(org.apache.flink.table.catalog.DataTypeFactory) AsyncLookupJoinWithCalcRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinWithCalcRunner) AsyncFunction(org.apache.flink.streaming.api.functions.async.AsyncFunction) LookupJoinCodeGenerator(org.apache.flink.table.planner.codegen.LookupJoinCodeGenerator) RowData(org.apache.flink.table.data.RowData) TableFunctionResultFuture(org.apache.flink.table.runtime.collector.TableFunctionResultFuture) FlatMapFunction(org.apache.flink.api.common.functions.FlatMapFunction)

Example 2 with AsyncLookupJoinRunner

use of org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner in project flink by apache.

the class AsyncLookupJoinHarnessTest method createHarness.

// ---------------------------------------------------------------------------------
@SuppressWarnings({ "unchecked", "rawtypes" })
private OneInputStreamOperatorTestHarness<RowData, RowData> createHarness(JoinType joinType, FilterOnTable filterOnTable) throws Exception {
    RichAsyncFunction<RowData, RowData> joinRunner;
    boolean isLeftJoin = joinType == JoinType.LEFT_JOIN;
    if (filterOnTable == FilterOnTable.WITHOUT_FILTER) {
        joinRunner = new AsyncLookupJoinRunner(new GeneratedFunctionWrapper(new TestingFetcherFunction()), fetcherConverter, new GeneratedResultFutureWrapper<>(new TestingFetcherResultFuture()), rightRowSerializer, isLeftJoin, ASYNC_BUFFER_CAPACITY);
    } else {
        joinRunner = new AsyncLookupJoinWithCalcRunner(new GeneratedFunctionWrapper(new TestingFetcherFunction()), fetcherConverter, new GeneratedFunctionWrapper<>(new CalculateOnTemporalTable()), new GeneratedResultFutureWrapper<>(new TestingFetcherResultFuture()), rightRowSerializer, isLeftJoin, ASYNC_BUFFER_CAPACITY);
    }
    return new OneInputStreamOperatorTestHarness<>(new AsyncWaitOperatorFactory<>(joinRunner, ASYNC_TIMEOUT_MS, ASYNC_BUFFER_CAPACITY, AsyncDataStream.OutputMode.ORDERED), inSerializer);
}
Also used : AsyncLookupJoinRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner) GenericRowData(org.apache.flink.table.data.GenericRowData) RowData(org.apache.flink.table.data.RowData) GeneratedFunctionWrapper(org.apache.flink.table.runtime.generated.GeneratedFunctionWrapper) GeneratedResultFutureWrapper(org.apache.flink.table.runtime.generated.GeneratedResultFutureWrapper) OneInputStreamOperatorTestHarness(org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness) AsyncLookupJoinWithCalcRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinWithCalcRunner)

Example 3 with AsyncLookupJoinRunner

use of org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner in project flink by apache.

the class AsyncLookupJoinHarnessTest method testCloseAsyncLookupJoinRunner.

@Test
public void testCloseAsyncLookupJoinRunner() throws Exception {
    final AsyncLookupJoinRunner joinRunner = new AsyncLookupJoinRunner(new GeneratedFunctionWrapper(new TestingFetcherFunction()), fetcherConverter, new GeneratedResultFutureWrapper<>(new TestingFetcherResultFuture()), rightRowSerializer, true, 100);
    assertNull(joinRunner.getAllResultFutures());
    closeAsyncLookupJoinRunner(joinRunner);
    joinRunner.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
    joinRunner.open(new Configuration());
    assertNotNull(joinRunner.getAllResultFutures());
    closeAsyncLookupJoinRunner(joinRunner);
    joinRunner.open(new Configuration());
    joinRunner.asyncInvoke(row(1, "a"), new TestingFetcherResultFuture());
    assertNotNull(joinRunner.getAllResultFutures());
    closeAsyncLookupJoinRunner(joinRunner);
}
Also used : AsyncLookupJoinRunner(org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner) MockStreamingRuntimeContext(org.apache.flink.streaming.util.MockStreamingRuntimeContext) Configuration(org.apache.flink.configuration.Configuration) GeneratedFunctionWrapper(org.apache.flink.table.runtime.generated.GeneratedFunctionWrapper) Test(org.junit.Test)

Aggregations

AsyncLookupJoinRunner (org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner)3 RowData (org.apache.flink.table.data.RowData)2 GeneratedFunctionWrapper (org.apache.flink.table.runtime.generated.GeneratedFunctionWrapper)2 AsyncLookupJoinWithCalcRunner (org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinWithCalcRunner)2 FlatMapFunction (org.apache.flink.api.common.functions.FlatMapFunction)1 Configuration (org.apache.flink.configuration.Configuration)1 AsyncFunction (org.apache.flink.streaming.api.functions.async.AsyncFunction)1 AsyncWaitOperatorFactory (org.apache.flink.streaming.api.operators.async.AsyncWaitOperatorFactory)1 MockStreamingRuntimeContext (org.apache.flink.streaming.util.MockStreamingRuntimeContext)1 OneInputStreamOperatorTestHarness (org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness)1 DataTypeFactory (org.apache.flink.table.catalog.DataTypeFactory)1 GenericRowData (org.apache.flink.table.data.GenericRowData)1 DataStructureConverter (org.apache.flink.table.data.conversion.DataStructureConverter)1 LookupJoinCodeGenerator (org.apache.flink.table.planner.codegen.LookupJoinCodeGenerator)1 TableFunctionResultFuture (org.apache.flink.table.runtime.collector.TableFunctionResultFuture)1 GeneratedResultFutureWrapper (org.apache.flink.table.runtime.generated.GeneratedResultFutureWrapper)1 RowType (org.apache.flink.table.types.logical.RowType)1 Test (org.junit.Test)1