Examples with DataInputStatus org.apache.flink.streaming.runtime.io.DataInputStatus used on opensource projects

Example 6 with DataInputStatus

use of org.apache.flink.streaming.runtime.io.DataInputStatus in project flink by apache.

the class LargeSortingDataInputITCase method intKeySorting.

@Test
public void intKeySorting() throws Exception {
    int numberOfRecords = 500_000;
    GeneratedRecordsDataInput input = new GeneratedRecordsDataInput(numberOfRecords, 0);
    KeySelector<Tuple3<Integer, String, byte[]>, Integer> keySelector = value -> value.f0;
    try (MockEnvironment environment = MockEnvironment.builder().build();
        SortingDataInput<Tuple3<Integer, String, byte[]>, Integer> sortingDataInput = new SortingDataInput<>(input, GeneratedRecordsDataInput.SERIALIZER, new IntSerializer(), keySelector, environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new DummyInvokable(), new ExecutionConfig())) {
        DataInputStatus inputStatus;
        VerifyingOutput<Integer> output = new VerifyingOutput<>(keySelector);
        do {
            inputStatus = sortingDataInput.emitNext(output);
        } while (inputStatus != DataInputStatus.END_OF_INPUT);
        assertThat(output.getSeenRecords(), equalTo(numberOfRecords));
    }
}

Example 7 with DataInputStatus

use of org.apache.flink.streaming.runtime.io.DataInputStatus in project flink by apache.

the class MultiInputSortingDataInputsTest method watermarkPropagation.

@Test
@SuppressWarnings("unchecked")
public void watermarkPropagation() throws Exception {
    CollectingDataOutput<Object> collectingDataOutput = new CollectingDataOutput<>();
    List<StreamElement> elements1 = Arrays.asList(new StreamRecord<>(2, 3), new Watermark(3), new StreamRecord<>(3, 3), new Watermark(7));
    List<StreamElement> elements2 = Arrays.asList(new StreamRecord<>(0, 3), new Watermark(1), new StreamRecord<>(1, 3), new Watermark(3));
    CollectionDataInput<Integer> dataInput1 = new CollectionDataInput<>(elements1, 0);
    CollectionDataInput<Integer> dataInput2 = new CollectionDataInput<>(elements2, 1);
    KeySelector<Integer, Integer> keySelector = value -> value;
    try (MockEnvironment environment = MockEnvironment.builder().build()) {
        SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(new DummyInvokable(), new StreamTaskInput[] { dataInput1, dataInput2 }, new KeySelector[] { keySelector, keySelector }, new TypeSerializer[] { new IntSerializer(), new IntSerializer() }, new IntSerializer(), new StreamTaskInput[0], environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new ExecutionConfig());
        StreamTaskInput<?>[] sortingDataInputs = selectableSortingInputs.getSortedInputs();
        try (StreamTaskInput<Object> input1 = (StreamTaskInput<Object>) sortingDataInputs[0];
            StreamTaskInput<Object> input2 = (StreamTaskInput<Object>) sortingDataInputs[1]) {
            MultipleInputSelectionHandler selectionHandler = new MultipleInputSelectionHandler(selectableSortingInputs.getInputSelectable(), 2);
            StreamMultipleInputProcessor processor = new StreamMultipleInputProcessor(selectionHandler, new StreamOneInputProcessor[] { new StreamOneInputProcessor<>(input1, collectingDataOutput, new DummyOperatorChain()), new StreamOneInputProcessor<>(input2, collectingDataOutput, new DummyOperatorChain()) });
            DataInputStatus inputStatus;
            do {
                inputStatus = processor.processInput();
            } while (inputStatus != DataInputStatus.END_OF_INPUT);
        }
    }
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(0, 3), new StreamRecord<>(1, 3), // watermark from the second input
    new Watermark(3), new StreamRecord<>(2, 3), new StreamRecord<>(3, 3), // watermark from the first input
    new Watermark(7))));
}

Example 8 with DataInputStatus

use of org.apache.flink.streaming.runtime.io.DataInputStatus in project flink by apache.

the class MultiInputSortingDataInputsTest method simpleFixedLengthKeySorting.

@Test
@SuppressWarnings("unchecked")
public void simpleFixedLengthKeySorting() throws Exception {
    CollectingDataOutput<Object> collectingDataOutput = new CollectingDataOutput<>();
    List<StreamElement> elements = Arrays.asList(new StreamRecord<>(1, 3), new StreamRecord<>(1, 1), new StreamRecord<>(2, 1), new StreamRecord<>(2, 3), new StreamRecord<>(1, 2), new StreamRecord<>(2, 2), Watermark.MAX_WATERMARK);
    CollectionDataInput<Integer> dataInput1 = new CollectionDataInput<>(elements, 0);
    CollectionDataInput<Integer> dataInput2 = new CollectionDataInput<>(elements, 1);
    KeySelector<Integer, Integer> keySelector = value -> value;
    try (MockEnvironment environment = MockEnvironment.builder().build()) {
        SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(new DummyInvokable(), new StreamTaskInput[] { dataInput1, dataInput2 }, new KeySelector[] { keySelector, keySelector }, new TypeSerializer[] { new IntSerializer(), new IntSerializer() }, new IntSerializer(), new StreamTaskInput[0], environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new ExecutionConfig());
        StreamTaskInput<?>[] sortingDataInputs = selectableSortingInputs.getSortedInputs();
        try (StreamTaskInput<Object> input1 = (StreamTaskInput<Object>) sortingDataInputs[0];
            StreamTaskInput<Object> input2 = (StreamTaskInput<Object>) sortingDataInputs[1]) {
            MultipleInputSelectionHandler selectionHandler = new MultipleInputSelectionHandler(selectableSortingInputs.getInputSelectable(), 2);
            StreamMultipleInputProcessor processor = new StreamMultipleInputProcessor(selectionHandler, new StreamOneInputProcessor[] { new StreamOneInputProcessor<>(input1, collectingDataOutput, new DummyOperatorChain()), new StreamOneInputProcessor<>(input2, collectingDataOutput, new DummyOperatorChain()) });
            DataInputStatus inputStatus;
            do {
                inputStatus = processor.processInput();
            } while (inputStatus != DataInputStatus.END_OF_INPUT);
        }
    }
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(1, 1), new StreamRecord<>(1, 1), new StreamRecord<>(1, 2), new StreamRecord<>(1, 2), new StreamRecord<>(1, 3), new StreamRecord<>(1, 3), new StreamRecord<>(2, 1), new StreamRecord<>(2, 1), new StreamRecord<>(2, 2), new StreamRecord<>(2, 2), new StreamRecord<>(2, 3), // max watermark from one of the inputs
    Watermark.MAX_WATERMARK, new StreamRecord<>(2, 3), // max watermark from the other input
    Watermark.MAX_WATERMARK)));
}

Example 9 with DataInputStatus

use of org.apache.flink.streaming.runtime.io.DataInputStatus in project flink by apache.

the class SortingDataInputTest method simpleFixedLengthKeySorting.

@Test
public void simpleFixedLengthKeySorting() throws Exception {
    CollectingDataOutput<Integer> collectingDataOutput = new CollectingDataOutput<>();
    CollectionDataInput<Integer> input = new CollectionDataInput<>(Arrays.asList(new StreamRecord<>(1, 3), new StreamRecord<>(1, 1), new StreamRecord<>(2, 1), new StreamRecord<>(2, 3), new StreamRecord<>(1, 2), new StreamRecord<>(2, 2)));
    MockEnvironment environment = MockEnvironment.builder().build();
    SortingDataInput<Integer, Integer> sortingDataInput = new SortingDataInput<>(input, new IntSerializer(), new IntSerializer(), (KeySelector<Integer, Integer>) value -> value, environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new DummyInvokable(), new ExecutionConfig());
    DataInputStatus inputStatus;
    do {
        inputStatus = sortingDataInput.emitNext(collectingDataOutput);
    } while (inputStatus != DataInputStatus.END_OF_INPUT);
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(1, 1), new StreamRecord<>(1, 2), new StreamRecord<>(1, 3), new StreamRecord<>(2, 1), new StreamRecord<>(2, 2), new StreamRecord<>(2, 3))));
}

Example 10 with DataInputStatus

use of org.apache.flink.streaming.runtime.io.DataInputStatus in project flink by apache.

the class SortingDataInputTest method watermarkPropagation.

@Test
public void watermarkPropagation() throws Exception {
    CollectingDataOutput<Integer> collectingDataOutput = new CollectingDataOutput<>();
    CollectionDataInput<Integer> input = new CollectionDataInput<>(Arrays.asList(new StreamRecord<>(1, 3), new Watermark(1), new StreamRecord<>(1, 1), new Watermark(2), new StreamRecord<>(2, 1), new Watermark(3), new StreamRecord<>(2, 3), new Watermark(4), new StreamRecord<>(1, 2), new Watermark(5), new StreamRecord<>(2, 2), new Watermark(6)));
    MockEnvironment environment = MockEnvironment.builder().build();
    SortingDataInput<Integer, Integer> sortingDataInput = new SortingDataInput<>(input, new IntSerializer(), new IntSerializer(), (KeySelector<Integer, Integer>) value -> value, environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new DummyInvokable(), new ExecutionConfig());
    DataInputStatus inputStatus;
    do {
        inputStatus = sortingDataInput.emitNext(collectingDataOutput);
    } while (inputStatus != DataInputStatus.END_OF_INPUT);
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(1, 1), new StreamRecord<>(1, 2), new StreamRecord<>(1, 3), new StreamRecord<>(2, 1), new StreamRecord<>(2, 2), new StreamRecord<>(2, 3), new Watermark(6))));
}