Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 81 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamSourceOperatorLatencyMetricsTest method setupSourceOperator.

// ------------------------------------------------------------------------
@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator, ExecutionConfig executionConfig, Environment env, TimerService timerService) {
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateBackend(new MemoryStateBackend());
    cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
    cfg.setOperatorID(new OperatorID());
    try {
        MockStreamTask mockTask = new MockStreamTaskBuilder(env).setConfig(cfg).setExecutionConfig(executionConfig).setTimerService(timerService).build();
        operator.setProcessingTimeService(mockTask.getProcessingTimeServiceFactory().createProcessingTimeService(null));
        operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 82 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method createBarrier.

private CheckpointBarrier createBarrier(StreamTaskMailboxTestHarness<String> testHarness) {
    StreamConfig config = testHarness.getStreamTask().getConfiguration();
    CheckpointOptions checkpointOptions = CheckpointOptions.forConfig(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault(), config.isExactlyOnceCheckpointMode(), config.isUnalignedCheckpointsEnabled(), config.getAlignedCheckpointTimeout().toMillis());
    return new CheckpointBarrier(metaData.getCheckpointId(), metaData.getTimestamp(), checkpointOptions);
}

Example 83 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method configureChainedTestingStreamOperator.

// ==============================================================================================
// Utility functions and classes
// ==============================================================================================
private void configureChainedTestingStreamOperator(StreamConfig streamConfig, int numberChainedTasks) {
    Preconditions.checkArgument(numberChainedTasks >= 1, "The operator chain must at least " + "contain one operator.");
    TestingStreamOperator<Integer, Integer> previousOperator = new TestingStreamOperator<>();
    streamConfig.setStreamOperator(previousOperator);
    streamConfig.setOperatorID(new OperatorID(0L, 0L));
    // create the chain of operators
    Map<Integer, StreamConfig> chainedTaskConfigs = new HashMap<>(numberChainedTasks - 1);
    List<StreamEdge> outputEdges = new ArrayList<>(numberChainedTasks - 1);
    for (int chainedIndex = 1; chainedIndex < numberChainedTasks; chainedIndex++) {
        TestingStreamOperator<Integer, Integer> chainedOperator = new TestingStreamOperator<>();
        StreamConfig chainedConfig = new StreamConfig(new Configuration());
        chainedConfig.setupNetworkInputs(StringSerializer.INSTANCE);
        chainedConfig.setStreamOperator(chainedOperator);
        chainedConfig.setOperatorID(new OperatorID(0L, chainedIndex));
        chainedTaskConfigs.put(chainedIndex, chainedConfig);
        StreamEdge outputEdge = new StreamEdge(new StreamNode(chainedIndex - 1, null, null, (StreamOperator<?>) null, null, null), new StreamNode(chainedIndex, null, null, (StreamOperator<?>) null, null, null), 0, null, null);
        outputEdges.add(outputEdge);
    }
    streamConfig.setChainedOutputs(outputEdges);
    streamConfig.setTransitiveChainedTaskConfigs(chainedTaskConfigs);
}

Example 84 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method testOvertakingCheckpointBarriers.

/**
 * This test verifies that checkpoint barriers and barrier buffers work correctly with
 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
 * some inputs receive barriers from an earlier checkpoint, thereby blocking, then all inputs
 * receive barriers from a later checkpoint.
 */
@Test
public void testOvertakingCheckpointBarriers() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<>("Hello-1-1", initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>("Ciao-1-1", initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<>("Hello-1-1", initialTime));
    expectedOutput.add(new StreamRecord<>("Ciao-1-1", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Now give a later barrier to all inputs, this should unblock the first channel
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    expectedOutput.add(new CancelCheckpointMarker(0));
    expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Then give the earlier barrier, these should be ignored
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 85 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SourceStreamTaskTest method testInterruptionExceptionNotSwallowed.

private void testInterruptionExceptionNotSwallowed(InterruptedSource.ExceptionGenerator exceptionGenerator) throws Exception {
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, STRING_TYPE_INFO);
    CancelLockingSource.reset();
    testHarness.setupOperatorChain(new OperatorID(), new StreamSource<>(new InterruptedSource(exceptionGenerator))).chain(new OperatorID(), new TestBoundedOneInputStreamOperator("Operator1"), STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    testHarness.invoke();
    try {
        testHarness.waitForTaskCompletion();
    } catch (Exception e) {
        if (!ExceptionUtils.findThrowable(e, InterruptedException.class).isPresent()) {
            throw e;
        }
    }
}