Examples with StreamTestSingleInputGate org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate used on opensource projects

Example 1 with StreamTestSingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate in project flink by apache.

the class StreamTaskNetworkInputTest method testInputStatusAfterEndOfRecovery.

@Test
public void testInputStatusAfterEndOfRecovery() throws Exception {
    int numInputChannels = 2;
    LongSerializer inSerializer = LongSerializer.INSTANCE;
    StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
    DataOutput<Long> output = new NoOpDataOutput<>();
    Map<InputChannelInfo, TestRecordDeserializer> deserializers = createDeserializers(inputGate.getInputGate());
    StreamTaskInput<Long> input = new TestStreamTaskNetworkInput(inputGate, inSerializer, numInputChannels, deserializers);
    inputGate.sendElement(new StreamRecord<>(42L), 0);
    assertThat(input.emitNext(output), equalTo(DataInputStatus.MORE_AVAILABLE));
    inputGate.sendEvent(EndOfChannelStateEvent.INSTANCE, 0);
    assertThat(input.emitNext(output), equalTo(DataInputStatus.MORE_AVAILABLE));
    inputGate.sendEvent(EndOfChannelStateEvent.INSTANCE, 1);
    assertThat(input.emitNext(output), equalTo(DataInputStatus.END_OF_RECOVERY));
}

Example 2 with StreamTestSingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate in project flink by apache.

the class TwoInputStreamTaskTestHarness method initializeInputs.

@Override
protected void initializeInputs() {
    inputGates = new StreamTestSingleInputGate[numInputGates];
    List<StreamEdge> inPhysicalEdges = new LinkedList<>();
    StreamOperator<IN1> dummyOperator = new AbstractStreamOperator<IN1>() {

        private static final long serialVersionUID = 1L;
    };
    StreamNode sourceVertexDummy = new StreamNode(0, "default group", null, dummyOperator, "source dummy", SourceStreamTask.class);
    StreamNode targetVertexDummy = new StreamNode(1, "default group", null, dummyOperator, "target dummy", SourceStreamTask.class);
    for (int i = 0; i < numInputGates; i++) {
        switch(inputGateAssignment[i]) {
            case 1:
                {
                    inputGates[i] = new StreamTestSingleInputGate<>(numInputChannelsPerGate, i, inputSerializer1, bufferSize);
                    StreamEdge streamEdge = new StreamEdge(sourceVertexDummy, targetVertexDummy, 1, new BroadcastPartitioner<>(), null);
                    inPhysicalEdges.add(streamEdge);
                    break;
                }
            case 2:
                {
                    inputGates[i] = new StreamTestSingleInputGate<>(numInputChannelsPerGate, i, inputSerializer2, bufferSize);
                    StreamEdge streamEdge = new StreamEdge(sourceVertexDummy, targetVertexDummy, 2, new BroadcastPartitioner<>(), null);
                    inPhysicalEdges.add(streamEdge);
                    break;
                }
            default:
                throw new IllegalStateException("Wrong input gate assignment.");
        }
        this.mockEnv.addInputGate(inputGates[i].getInputGate());
    }
    streamConfig.setInPhysicalEdges(inPhysicalEdges);
    streamConfig.setNumberOfNetworkInputs(numInputGates);
    streamConfig.setupNetworkInputs(inputSerializer1, inputSerializer2);
}

Example 3 with StreamTestSingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate in project flink by apache.

the class StreamTaskNetworkInputTest method testSnapshotAfterEndOfPartition.

@Test
public void testSnapshotAfterEndOfPartition() throws Exception {
    int numInputChannels = 1;
    int channelId = 0;
    int checkpointId = 0;
    VerifyRecordsDataOutput<Long> output = new VerifyRecordsDataOutput<>();
    LongSerializer inSerializer = LongSerializer.INSTANCE;
    StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
    StreamTaskInput<Long> input = new StreamTaskNetworkInput<>(new CheckpointedInputGate(inputGate.getInputGate(), SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", new DummyCheckpointInvokable(), SystemClock.getInstance(), false, inputGate.getInputGate()), new SyncMailboxExecutor()), inSerializer, ioManager, new StatusWatermarkValve(numInputChannels), 0);
    inputGate.sendEvent(new CheckpointBarrier(checkpointId, 0L, CheckpointOptions.forCheckpointWithDefaultLocation().toUnaligned()), channelId);
    inputGate.sendElement(new StreamRecord<>(42L), channelId);
    assertHasNextElement(input, output);
    assertHasNextElement(input, output);
    assertEquals(1, output.getNumberOfEmittedRecords());
    // send EndOfPartitionEvent and ensure that deserializer has been released
    inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelId);
    input.emitNext(output);
    // now snapshot all inflight buffers
    CompletableFuture<Void> completableFuture = input.prepareSnapshot(ChannelStateWriter.NO_OP, checkpointId);
    completableFuture.join();
}

Example 4 with StreamTestSingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate in project flink by apache.

the class StreamTaskNetworkInputTest method testReleasingDeserializerTimely.

@Test
public void testReleasingDeserializerTimely() throws Exception {
    int numInputChannels = 2;
    LongSerializer inSerializer = LongSerializer.INSTANCE;
    StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
    DataOutput<Long> output = new NoOpDataOutput<>();
    Map<InputChannelInfo, TestRecordDeserializer> deserializers = createDeserializers(inputGate.getInputGate());
    Map<InputChannelInfo, TestRecordDeserializer> copiedDeserializers = new HashMap<>(deserializers);
    StreamTaskInput<Long> input = new TestStreamTaskNetworkInput(inputGate, inSerializer, numInputChannels, deserializers);
    for (InputChannelInfo channelInfo : inputGate.getInputGate().getChannelInfos()) {
        assertNotNull(deserializers.get(channelInfo));
        inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelInfo.getInputChannelIdx());
        input.emitNext(output);
        assertTrue(copiedDeserializers.get(channelInfo).isCleared());
        assertNull(deserializers.get(channelInfo));
    }
}