use of org.apache.flink.table.connector.sink.SinkFunctionProvider in project flink by apache.
the class HBaseDynamicTableFactoryTest method testParallelismOptions.
@Test
public void testParallelismOptions() {
Map<String, String> options = getAllOptions();
options.put("sink.parallelism", "2");
ResolvedSchema schema = ResolvedSchema.of(Column.physical(ROWKEY, STRING()));
DynamicTableSink sink = createTableSink(schema, options);
assertTrue(sink instanceof HBaseDynamicTableSink);
HBaseDynamicTableSink hbaseSink = (HBaseDynamicTableSink) sink;
SinkFunctionProvider provider = (SinkFunctionProvider) hbaseSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
assertEquals(2, (long) provider.getParallelism().get());
}
use of org.apache.flink.table.connector.sink.SinkFunctionProvider in project flink by apache.
the class HBaseDynamicTableFactoryTest method testParallelismOptions.
@Test
public void testParallelismOptions() {
Map<String, String> options = getAllOptions();
options.put("sink.parallelism", "2");
ResolvedSchema schema = ResolvedSchema.of(Column.physical(ROWKEY, STRING()));
DynamicTableSink sink = createTableSink(schema, options);
assertTrue(sink instanceof HBaseDynamicTableSink);
HBaseDynamicTableSink hbaseSink = (HBaseDynamicTableSink) sink;
SinkFunctionProvider provider = (SinkFunctionProvider) hbaseSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
assertEquals(2, (long) provider.getParallelism().get());
}
use of org.apache.flink.table.connector.sink.SinkFunctionProvider in project flink by apache.
the class JdbcDynamicTableSinkITCase method testFlushBufferWhenCheckpoint.
@Test
public void testFlushBufferWhenCheckpoint() throws Exception {
Map<String, String> options = new HashMap<>();
options.put("connector", "jdbc");
options.put("url", DB_URL);
options.put("table-name", OUTPUT_TABLE5);
options.put("sink.buffer-flush.interval", "0");
ResolvedSchema schema = ResolvedSchema.of(Column.physical("id", DataTypes.BIGINT().notNull()));
DynamicTableSink tableSink = createTableSink(schema, options);
SinkRuntimeProviderContext context = new SinkRuntimeProviderContext(false);
SinkFunctionProvider sinkProvider = (SinkFunctionProvider) tableSink.getSinkRuntimeProvider(context);
GenericJdbcSinkFunction<RowData> sinkFunction = (GenericJdbcSinkFunction<RowData>) sinkProvider.createSinkFunction();
sinkFunction.setRuntimeContext(new MockStreamingRuntimeContext(true, 1, 0));
sinkFunction.open(new Configuration());
sinkFunction.invoke(GenericRowData.of(1L), SinkContextUtil.forTimestamp(1));
sinkFunction.invoke(GenericRowData.of(2L), SinkContextUtil.forTimestamp(1));
check(new Row[] {}, DB_URL, OUTPUT_TABLE5, new String[] { "id" });
sinkFunction.snapshotState(new StateSnapshotContextSynchronousImpl(1, 1));
check(new Row[] { Row.of(1L), Row.of(2L) }, DB_URL, OUTPUT_TABLE5, new String[] { "id" });
sinkFunction.close();
}
use of org.apache.flink.table.connector.sink.SinkFunctionProvider in project flink by apache.
the class OracleTableSinkITCase method testFlushBufferWhenCheckpoint.
@Test
public void testFlushBufferWhenCheckpoint() throws Exception {
Map<String, String> options = new HashMap<>();
options.put("connector", "jdbc");
options.put("url", containerUrl);
options.put("table-name", OUTPUT_TABLE5);
options.put("sink.buffer-flush.interval", "0");
ResolvedSchema schema = ResolvedSchema.of(Column.physical("id", DataTypes.BIGINT().notNull()));
DynamicTableSink tableSink = createTableSink(schema, options);
SinkRuntimeProviderContext context = new SinkRuntimeProviderContext(false);
SinkFunctionProvider sinkProvider = (SinkFunctionProvider) tableSink.getSinkRuntimeProvider(context);
GenericJdbcSinkFunction<RowData> sinkFunction = (GenericJdbcSinkFunction<RowData>) sinkProvider.createSinkFunction();
sinkFunction.setRuntimeContext(new MockStreamingRuntimeContext(true, 1, 0));
sinkFunction.open(new Configuration());
sinkFunction.invoke(GenericRowData.of(1L), SinkContextUtil.forTimestamp(1));
sinkFunction.invoke(GenericRowData.of(2L), SinkContextUtil.forTimestamp(1));
check(new Row[] {}, containerUrl, OUTPUT_TABLE5, new String[] { "id" });
sinkFunction.snapshotState(new StateSnapshotContextSynchronousImpl(1, 1));
check(new Row[] { Row.of(1L), Row.of(2L) }, containerUrl, OUTPUT_TABLE5, new String[] { "id" });
sinkFunction.close();
}
use of org.apache.flink.table.connector.sink.SinkFunctionProvider in project flink by apache.
the class CommonExecSink method applySinkProvider.
private Transformation<?> applySinkProvider(Transformation<RowData> inputTransform, StreamExecutionEnvironment env, SinkRuntimeProvider runtimeProvider, int rowtimeFieldIndex, int sinkParallelism, ReadableConfig config) {
TransformationMetadata sinkMeta = createTransformationMeta(SINK_TRANSFORMATION, config);
if (runtimeProvider instanceof DataStreamSinkProvider) {
Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
final DataStreamSinkProvider provider = (DataStreamSinkProvider) runtimeProvider;
return provider.consumeDataStream(createProviderContext(), dataStream).getTransformation();
} else if (runtimeProvider instanceof TransformationSinkProvider) {
final TransformationSinkProvider provider = (TransformationSinkProvider) runtimeProvider;
return provider.createTransformation(new TransformationSinkProvider.Context() {
@Override
public Transformation<RowData> getInputTransformation() {
return inputTransform;
}
@Override
public int getRowtimeIndex() {
return rowtimeFieldIndex;
}
@Override
public Optional<String> generateUid(String name) {
return createProviderContext().generateUid(name);
}
});
} else if (runtimeProvider instanceof SinkFunctionProvider) {
final SinkFunction<RowData> sinkFunction = ((SinkFunctionProvider) runtimeProvider).createSinkFunction();
return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
} else if (runtimeProvider instanceof OutputFormatProvider) {
OutputFormat<RowData> outputFormat = ((OutputFormatProvider) runtimeProvider).createOutputFormat();
final SinkFunction<RowData> sinkFunction = new OutputFormatSinkFunction<>(outputFormat);
return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
} else if (runtimeProvider instanceof SinkProvider) {
Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
final Transformation<?> transformation = DataStreamSink.forSinkV1(dataStream, ((SinkProvider) runtimeProvider).createSink()).getTransformation();
transformation.setParallelism(sinkParallelism);
sinkMeta.fill(transformation);
return transformation;
} else if (runtimeProvider instanceof SinkV2Provider) {
Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
final Transformation<?> transformation = DataStreamSink.forSink(dataStream, ((SinkV2Provider) runtimeProvider).createSink()).getTransformation();
transformation.setParallelism(sinkParallelism);
sinkMeta.fill(transformation);
return transformation;
} else {
throw new TableException("Unsupported sink runtime provider.");
}
}
Aggregations