Examples with StreamOperatorFactory org.apache.flink.streaming.api.operators.StreamOperatorFactory used on opensource projects

Example 1 with StreamOperatorFactory

use of org.apache.flink.streaming.api.operators.StreamOperatorFactory in project flink by apache.

the class UpsertKafkaDynamicTableFactoryTest method testBufferedTableSink.

@SuppressWarnings("rawtypes")
@Test
public void testBufferedTableSink() {
    // Construct table sink using options and table sink factory.
    final DynamicTableSink actualSink = createTableSink(SINK_SCHEMA, getModifiedOptions(getFullSinkOptions(), options -> {
        options.put("sink.buffer-flush.max-rows", "100");
        options.put("sink.buffer-flush.interval", "1s");
    }));
    final DynamicTableSink expectedSink = createExpectedSink(SINK_SCHEMA.toPhysicalRowDataType(), keyEncodingFormat, valueEncodingFormat, SINK_KEY_FIELDS, SINK_VALUE_FIELDS, null, SINK_TOPIC, UPSERT_KAFKA_SINK_PROPERTIES, DeliveryGuarantee.AT_LEAST_ONCE, new SinkBufferFlushMode(100, 1000L), null);
    // Test sink format.
    final KafkaDynamicSink actualUpsertKafkaSink = (KafkaDynamicSink) actualSink;
    assertEquals(expectedSink, actualSink);
    // Test kafka producer.
    DynamicTableSink.SinkRuntimeProvider provider = actualUpsertKafkaSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
    assertThat(provider, instanceOf(DataStreamSinkProvider.class));
    final DataStreamSinkProvider sinkProvider = (DataStreamSinkProvider) provider;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    sinkProvider.consumeDataStream(n -> Optional.empty(), env.fromElements(new BinaryRowData(1)));
    final StreamOperatorFactory<?> sinkOperatorFactory = env.getStreamGraph().getStreamNodes().stream().filter(n -> n.getOperatorName().contains("Writer")).findFirst().orElseThrow(() -> new RuntimeException("Expected operator with name Sink in stream graph.")).getOperatorFactory();
    assertThat(sinkOperatorFactory, instanceOf(SinkWriterOperatorFactory.class));
    org.apache.flink.api.connector.sink2.Sink sink = ((SinkWriterOperatorFactory) sinkOperatorFactory).getSink();
    assertThat(sink, instanceOf(ReducingUpsertSink.class));
}

Example 2 with StreamOperatorFactory

use of org.apache.flink.streaming.api.operators.StreamOperatorFactory in project flink by apache.

the class StreamingJobGraphGenerator method preValidate.

@SuppressWarnings("deprecation")
private void preValidate() {
    CheckpointConfig checkpointConfig = streamGraph.getCheckpointConfig();
    if (checkpointConfig.isCheckpointingEnabled()) {
        // temporarily forbid checkpointing for iterative jobs
        if (streamGraph.isIterative() && !checkpointConfig.isForceCheckpointing()) {
            throw new UnsupportedOperationException("Checkpointing is currently not supported by default for iterative jobs, as we cannot guarantee exactly once semantics. " + "State checkpoints happen normally, but records in-transit during the snapshot will be lost upon failure. " + "\nThe user can force enable state checkpoints with the reduced guarantees by calling: env.enableCheckpointing(interval,true)");
        }
        if (streamGraph.isIterative() && checkpointConfig.isUnalignedCheckpointsEnabled() && !checkpointConfig.isForceUnalignedCheckpoints()) {
            throw new UnsupportedOperationException("Unaligned Checkpoints are currently not supported for iterative jobs, " + "as rescaling would require alignment (in addition to the reduced checkpointing guarantees)." + "\nThe user can force Unaligned Checkpoints by using 'execution.checkpointing.unaligned.forced'");
        }
        if (checkpointConfig.isUnalignedCheckpointsEnabled() && !checkpointConfig.isForceUnalignedCheckpoints() && streamGraph.getStreamNodes().stream().anyMatch(this::hasCustomPartitioner)) {
            throw new UnsupportedOperationException("Unaligned checkpoints are currently not supported for custom partitioners, " + "as rescaling is not guaranteed to work correctly." + "\nThe user can force Unaligned Checkpoints by using 'execution.checkpointing.unaligned.forced'");
        }
        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        for (StreamNode node : streamGraph.getStreamNodes()) {
            StreamOperatorFactory operatorFactory = node.getOperatorFactory();
            if (operatorFactory != null) {
                Class<?> operatorClass = operatorFactory.getStreamOperatorClass(classLoader);
                if (InputSelectable.class.isAssignableFrom(operatorClass)) {
                    throw new UnsupportedOperationException("Checkpointing is currently not supported for operators that implement InputSelectable:" + operatorClass.getName());
                }
            }
        }
    }
    if (checkpointConfig.isUnalignedCheckpointsEnabled() && getCheckpointingMode(checkpointConfig) != CheckpointingMode.EXACTLY_ONCE) {
        LOG.warn("Unaligned checkpoints can only be used with checkpointing mode EXACTLY_ONCE");
        checkpointConfig.enableUnalignedCheckpoints(false);
    }
}

Example 3 with StreamOperatorFactory

use of org.apache.flink.streaming.api.operators.StreamOperatorFactory in project flink by apache.

the class StreamingJobGraphGeneratorTest method testCoordinatedOperator.

@Test
public void testCoordinatedOperator() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source = env.fromSource(new MockSource(Boundedness.BOUNDED, 1), WatermarkStrategy.noWatermarks(), "TestSource");
    source.addSink(new DiscardingSink<>());
    StreamGraph streamGraph = env.getStreamGraph();
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
    // There should be only one job vertex.
    assertEquals(1, jobGraph.getNumberOfVertices());
    JobVertex jobVertex = jobGraph.getVerticesAsArray()[0];
    List<SerializedValue<OperatorCoordinator.Provider>> coordinatorProviders = jobVertex.getOperatorCoordinators();
    // There should be only one coordinator provider.
    assertEquals(1, coordinatorProviders.size());
    // The invokable class should be SourceOperatorStreamTask.
    final ClassLoader classLoader = getClass().getClassLoader();
    assertEquals(SourceOperatorStreamTask.class, jobVertex.getInvokableClass(classLoader));
    StreamOperatorFactory operatorFactory = new StreamConfig(jobVertex.getConfiguration()).getStreamOperatorFactory(classLoader);
    assertTrue(operatorFactory instanceof SourceOperatorFactory);
}