Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 46 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class TwoInputStreamTaskTest method testOvertakingCheckpointBarriers.

/**
 * This test verifies that checkpoint barriers and barrier buffers work correctly with
 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
 * some inputs receive barriers from an earlier checkpoint, thereby blocking, then all inputs
 * receive barriers from a later checkpoint.
 */
@Test
public void testOvertakingCheckpointBarriers() throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<>(42, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>(1337, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<>("42", initialTime));
    expectedOutput.add(new StreamRecord<>("1337", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Now give a later barrier to all inputs, this should unblock the first channel
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    expectedOutput.add(new CancelCheckpointMarker(0));
    expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Then give the earlier barrier, these should be ignored
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 47 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class BoltWrapperTest method testWrapper.

@SuppressWarnings({ "rawtypes", "unchecked" })
private void testWrapper(final int numberOfAttributes) throws Exception {
    assert ((-1 <= numberOfAttributes) && (numberOfAttributes <= 25));
    Tuple flinkTuple = null;
    String rawTuple = null;
    if (numberOfAttributes == -1) {
        rawTuple = "test";
    } else {
        flinkTuple = Tuple.getTupleClass(numberOfAttributes).newInstance();
    }
    final String[] schema;
    if (numberOfAttributes == -1) {
        schema = new String[1];
    } else {
        schema = new String[numberOfAttributes];
    }
    for (int i = 0; i < schema.length; ++i) {
        schema[i] = "a" + i;
    }
    final StreamRecord record = mock(StreamRecord.class);
    if (numberOfAttributes == -1) {
        when(record.getValue()).thenReturn(rawTuple);
    } else {
        when(record.getValue()).thenReturn(flinkTuple);
    }
    final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class);
    when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class));
    when(taskContext.getTaskName()).thenReturn("name");
    when(taskContext.getMetricGroup()).thenReturn(new UnregisteredMetricsGroup());
    final IRichBolt bolt = mock(IRichBolt.class);
    final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer();
    declarer.declare(new Fields(schema));
    PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
    final BoltWrapper wrapper = new BoltWrapper(bolt, (Fields) null);
    wrapper.setup(createMockStreamTask(), new StreamConfig(new Configuration()), mock(Output.class));
    wrapper.open();
    wrapper.processElement(record);
    if (numberOfAttributes == -1) {
        verify(bolt).execute(eq(new StormTuple<String>(rawTuple, null, -1, null, null, MessageId.makeUnanchored())));
    } else {
        verify(bolt).execute(eq(new StormTuple<Tuple>(flinkTuple, null, -1, null, null, MessageId.makeUnanchored())));
    }
}

Example 48 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class BoltWrapperTest method createMockStreamTask.

public static StreamTask<?, ?> createMockStreamTask(ExecutionConfig execConfig) {
    Environment env = mock(Environment.class);
    when(env.getTaskInfo()).thenReturn(new TaskInfo("Mock Task", 1, 0, 1, 0));
    when(env.getUserClassLoader()).thenReturn(BoltWrapperTest.class.getClassLoader());
    when(env.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
    when(env.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
    StreamTask<?, ?> mockTask = mock(StreamTask.class);
    when(mockTask.getCheckpointLock()).thenReturn(new Object());
    when(mockTask.getConfiguration()).thenReturn(new StreamConfig(new Configuration()));
    when(mockTask.getEnvironment()).thenReturn(env);
    when(mockTask.getExecutionConfig()).thenReturn(execConfig);
    return mockTask;
}

Example 49 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTask method invoke.

@Override
public final void invoke() throws Exception {
    boolean disposed = false;
    try {
        // -------- Initialize ---------
        LOG.debug("Initializing {}.", getName());
        asyncOperationsThreadPool = Executors.newCachedThreadPool();
        configuration = new StreamConfig(getTaskConfiguration());
        stateBackend = createStateBackend();
        accumulatorMap = getEnvironment().getAccumulatorRegistry().getUserMap();
        // if the clock is not already set, then assign a default TimeServiceProvider
        if (timerService == null) {
            ThreadFactory timerThreadFactory = new DispatcherThreadFactory(TRIGGER_THREAD_GROUP, "Time Trigger for " + getName());
            timerService = new SystemProcessingTimeService(this, getCheckpointLock(), timerThreadFactory);
        }
        operatorChain = new OperatorChain<>(this);
        headOperator = operatorChain.getHeadOperator();
        // task specific initialization
        init();
        // save the work of reloading state, etc, if the task is already canceled
        if (canceled) {
            throw new CancelTaskException();
        }
        // -------- Invoke --------
        LOG.debug("Invoking {}", getName());
        // executed before all operators are opened
        synchronized (lock) {
            // both the following operations are protected by the lock
            // so that we avoid race conditions in the case that initializeState()
            // registers a timer, that fires before the open() is called.
            initializeState();
            openAllOperators();
        }
        // final check to exit early before starting to run
        if (canceled) {
            throw new CancelTaskException();
        }
        // let the task do its work
        isRunning = true;
        run();
        // make sure the "clean shutdown" is not attempted
        if (canceled) {
            throw new CancelTaskException();
        }
        // make sure all timers finish and no new timers can come
        timerService.quiesceAndAwaitPending();
        LOG.debug("Finished task {}", getName());
        // at the same time, this makes sure that during any "regular" exit where still
        synchronized (lock) {
            isRunning = false;
            // this is part of the main logic, so if this fails, the task is considered failed
            closeAllOperators();
        }
        LOG.debug("Closed operators for task {}", getName());
        // make sure all buffered data is flushed
        operatorChain.flushOutputs();
        // make an attempt to dispose the operators such that failures in the dispose call
        // still let the computation fail
        tryDisposeAllOperators();
        disposed = true;
    } finally {
        // clean up everything we initialized
        isRunning = false;
        // stop all timers and threads
        if (timerService != null) {
            try {
                timerService.shutdownService();
            } catch (Throwable t) {
                // catch and log the exception to not replace the original exception
                LOG.error("Could not shut down timer service", t);
            }
        }
        // stop all asynchronous checkpoint threads
        try {
            cancelables.close();
            shutdownAsyncThreads();
        } catch (Throwable t) {
            // catch and log the exception to not replace the original exception
            LOG.error("Could not shut down async checkpoint threads", t);
        }
        // we must! perform this cleanup
        try {
            cleanup();
        } catch (Throwable t) {
            // catch and log the exception to not replace the original exception
            LOG.error("Error during cleanup of stream task", t);
        }
        // if the operators were not disposed before, do a hard dispose
        if (!disposed) {
            disposeAllOperators();
        }
        // release the output resources. this method should never fail.
        if (operatorChain != null) {
            operatorChain.releaseOutputs();
        }
    }
}

Example 50 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OperatorChain method createOutputCollector.

// ------------------------------------------------------------------------
//  initialization utilities
// ------------------------------------------------------------------------
private <T> Output<StreamRecord<T>> createOutputCollector(StreamTask<?, ?> containingTask, StreamConfig operatorConfig, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, Map<StreamEdge, RecordWriterOutput<?>> streamOutputs, List<StreamOperator<?>> allOperators) {
    List<Tuple2<Output<StreamRecord<T>>, StreamEdge>> allOutputs = new ArrayList<>(4);
    // create collectors for the network outputs
    for (StreamEdge outputEdge : operatorConfig.getNonChainedOutputs(userCodeClassloader)) {
        @SuppressWarnings("unchecked") RecordWriterOutput<T> output = (RecordWriterOutput<T>) streamOutputs.get(outputEdge);
        allOutputs.add(new Tuple2<Output<StreamRecord<T>>, StreamEdge>(output, outputEdge));
    }
    // Create collectors for the chained outputs
    for (StreamEdge outputEdge : operatorConfig.getChainedOutputs(userCodeClassloader)) {
        int outputId = outputEdge.getTargetId();
        StreamConfig chainedOpConfig = chainedConfigs.get(outputId);
        Output<StreamRecord<T>> output = createChainedOperator(containingTask, chainedOpConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperators, outputEdge.getOutputTag());
        allOutputs.add(new Tuple2<>(output, outputEdge));
    }
    // if there are multiple outputs, or the outputs are directed, we need to
    // wrap them as one output
    List<OutputSelector<T>> selectors = operatorConfig.getOutputSelectors(userCodeClassloader);
    if (selectors == null || selectors.isEmpty()) {
        // simple path, no selector necessary
        if (allOutputs.size() == 1) {
            return allOutputs.get(0).f0;
        } else {
            // send to N outputs. Note that this includes teh special case
            // of sending to zero outputs
            @SuppressWarnings({ "unchecked", "rawtypes" }) Output<StreamRecord<T>>[] asArray = new Output[allOutputs.size()];
            for (int i = 0; i < allOutputs.size(); i++) {
                asArray[i] = allOutputs.get(i).f0;
            }
            // otherwise multi-chaining would not work correctly.
            if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
                return new CopyingBroadcastingOutputCollector<>(asArray, this);
            } else {
                return new BroadcastingOutputCollector<>(asArray, this);
            }
        }
    } else {
        // otherwise multi-chaining would not work correctly.
        if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
            return new CopyingDirectedOutput<>(selectors, allOutputs);
        } else {
            return new DirectedOutput<>(selectors, allOutputs);
        }
    }
}