Examples with CollectStreamSink org.apache.flink.streaming.api.operators.collect.CollectStreamSink used on opensource projects

Example 1 with CollectStreamSink

use of org.apache.flink.streaming.api.operators.collect.CollectStreamSink in project flink by apache.

the class DataStream method executeAndCollectWithClient.

ClientAndIterator<T> executeAndCollectWithClient(String jobExecutionName) throws Exception {
    TypeSerializer<T> serializer = getType().createSerializer(getExecutionEnvironment().getConfig());
    String accumulatorName = "dataStreamCollect_" + UUID.randomUUID().toString();
    StreamExecutionEnvironment env = getExecutionEnvironment();
    CollectSinkOperatorFactory<T> factory = new CollectSinkOperatorFactory<>(serializer, accumulatorName);
    CollectSinkOperator<T> operator = (CollectSinkOperator<T>) factory.getOperator();
    CollectResultIterator<T> iterator = new CollectResultIterator<>(operator.getOperatorIdFuture(), serializer, accumulatorName, env.getCheckpointConfig());
    CollectStreamSink<T> sink = new CollectStreamSink<>(this, factory);
    sink.name("Data stream collect sink");
    env.addOperator(sink.getTransformation());
    final JobClient jobClient = env.executeAsync(jobExecutionName);
    iterator.setJobClient(jobClient);
    return new ClientAndIterator<>(jobClient, iterator);
}

Example 2 with CollectStreamSink

use of org.apache.flink.streaming.api.operators.collect.CollectStreamSink in project flink by apache.

the class CollectDynamicSink method getSinkRuntimeProvider.

@Override
public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
    return (DataStreamSinkProvider) (providerContext, inputStream) -> {
        final CheckpointConfig checkpointConfig = inputStream.getExecutionEnvironment().getCheckpointConfig();
        final ExecutionConfig config = inputStream.getExecutionConfig();
        final TypeSerializer<RowData> externalSerializer = InternalTypeInfo.<RowData>of(consumedDataType.getLogicalType()).createSerializer(config);
        final String accumulatorName = tableIdentifier.getObjectName();
        final CollectSinkOperatorFactory<RowData> factory = new CollectSinkOperatorFactory<>(externalSerializer, accumulatorName, maxBatchSize, socketTimeout);
        final CollectSinkOperator<RowData> operator = (CollectSinkOperator<RowData>) factory.getOperator();
        iterator = new CollectResultIterator<>(operator.getOperatorIdFuture(), externalSerializer, accumulatorName, checkpointConfig);
        converter = context.createDataStructureConverter(consumedDataType);
        converter.open(RuntimeConverter.Context.create(classLoader));
        final CollectStreamSink<RowData> sink = new CollectStreamSink<>(inputStream, factory);
        providerContext.generateUid(COLLECT_TRANSFORMATION).ifPresent(sink::uid);
        return sink.name("Collect table sink");
    };
}