Examples with AddSplitEvent org.apache.flink.runtime.source.event.AddSplitEvent used on opensource projects

Example 1 with AddSplitEvent

use of org.apache.flink.runtime.source.event.AddSplitEvent in project flink by apache.

the class SourceOperatorStreamTaskTest method getAndMaybeAssignSplit.

private MockSourceSplit getAndMaybeAssignSplit(StreamTaskMailboxTestHarness<Integer> testHarness) throws Exception {
    List<MockSourceSplit> assignedSplits = getSourceReaderFromTask(testHarness).getAssignedSplits();
    if (assignedSplits.isEmpty()) {
        // Prepare the source split and assign it to the source reader.
        MockSourceSplit split = new MockSourceSplit(0, 0);
        // Assign the split to the source reader.
        AddSplitEvent<MockSourceSplit> addSplitEvent = new AddSplitEvent<>(Collections.singletonList(split), new MockSourceSplitSerializer());
        testHarness.getStreamTask().dispatchOperatorEvent(OPERATOR_ID, new SerializedValue<>(addSplitEvent));
        // Run the task until the split assignment is done.
        while (assignedSplits.isEmpty()) {
            testHarness.getStreamTask().runMailboxStep();
        }
        // Need to mark the source reader as available for further processing.
        getSourceReaderFromTask(testHarness).markAvailable();
    }
    // The source reader already has an assigned split, just return it
    return assignedSplits.get(0);
}

Example 2 with AddSplitEvent

use of org.apache.flink.runtime.source.event.AddSplitEvent in project flink by apache.

the class SourceCoordinatorContextTest method testAssignSplits.

@SuppressWarnings("unchecked")
private void testAssignSplits(boolean fromCoordinatorExecutor) throws Exception {
    sourceReady();
    registerReaders();
    // Assign splits to the readers.
    SplitsAssignment<MockSourceSplit> splitsAssignment = getSplitsAssignment(2, 0);
    if (fromCoordinatorExecutor) {
        coordinatorExecutor.submit(() -> context.assignSplits(splitsAssignment)).get();
    } else {
        context.assignSplits(splitsAssignment);
    }
    // The tracker should have recorded the assignments.
    verifyAssignment(Collections.singletonList("0"), splitSplitAssignmentTracker.uncheckpointedAssignments().get(0));
    verifyAssignment(Arrays.asList("1", "2"), splitSplitAssignmentTracker.uncheckpointedAssignments().get(1));
    // The OperatorCoordinatorContext should have received the event sending call.
    assertEquals("There should be two events sent to the subtasks.", 2, receivingTasks.getNumberOfSentEvents());
    // Assert the events to subtask0.
    List<OperatorEvent> eventsToSubtask0 = receivingTasks.getSentEventsForSubtask(0);
    assertEquals(1, eventsToSubtask0.size());
    OperatorEvent event = eventsToSubtask0.get(0);
    assertTrue(event instanceof AddSplitEvent);
    verifyAssignment(Collections.singletonList("0"), ((AddSplitEvent<MockSourceSplit>) event).splits(new MockSourceSplitSerializer()));
}

Example 3 with AddSplitEvent

use of org.apache.flink.runtime.source.event.AddSplitEvent in project flink by apache.

the class MultipleInputStreamTaskTest method addSourceRecords.

static void addSourceRecords(StreamTaskMailboxTestHarness<String> testHarness, int sourceId, Boundedness boundedness, int... records) throws Exception {
    OperatorID sourceOperatorID = getSourceOperatorID(testHarness, sourceId);
    // Prepare the source split and assign it to the source reader.
    MockSourceSplit split = new MockSourceSplit(0, 0, boundedness == Boundedness.BOUNDED ? records.length : Integer.MAX_VALUE);
    for (int record : records) {
        split.addRecord(record);
    }
    // Assign the split to the source reader.
    AddSplitEvent<MockSourceSplit> addSplitEvent = new AddSplitEvent<>(Collections.singletonList(split), new MockSourceSplitSerializer());
    testHarness.getStreamTask().dispatchOperatorEvent(sourceOperatorID, new SerializedValue<>(addSplitEvent));
}

Example 4 with AddSplitEvent

use of org.apache.flink.runtime.source.event.AddSplitEvent in project flink by apache.

the class SourceOperatorAlignmentTest method testWatermarkAlignmentWithIdleness.

@Test
public void testWatermarkAlignmentWithIdleness() throws Exception {
    // we use a separate context, because we need to enable idleness
    try (SourceOperatorTestContext context = new SourceOperatorTestContext(true, WatermarkStrategy.forGenerator(ctx -> new PunctuatedGenerator(PunctuatedGenerator.GenerationMode.ODD)).withWatermarkAlignment("group1", Duration.ofMillis(100), Duration.ofMillis(1)).withTimestampAssigner((r, t) -> r))) {
        final SourceOperator<Integer, MockSourceSplit> operator = context.getOperator();
        operator.initializeState(context.createStateContext());
        operator.open();
        MockSourceSplit newSplit = new MockSourceSplit(2);
        int record1 = 1;
        newSplit.addRecord(record1);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
        List<Integer> expectedOutput = new ArrayList<>();
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record1);
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
        // source becomes idle, it should report Long.MAX_VALUE as the watermark
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, Long.MAX_VALUE);
        // it is easier to create a new split than add records the old one. The old one is
        // serialized, when sending the AddSplitEvent, so it is not as easy as
        // newSplit.addRecord
        newSplit = new MockSourceSplit(3);
        // even timestamp -> no watermarks
        int record2 = 2;
        newSplit.addRecord(record2);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record2);
        context.getTimeService().advance(1);
        // becomes active again, should go back to the previously emitted
        // watermark, as the record2 does not emit watermarks
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
    }
}