Examples with SourceOperatorFactory org.apache.flink.streaming.api.operators.SourceOperatorFactory used on opensource projects

Example 1 with SourceOperatorFactory

use of org.apache.flink.streaming.api.operators.SourceOperatorFactory in project flink by apache.

the class MultipleInputStreamTaskTest method testTriggeringStopWithSavepointWithDrain.

@Test
public void testTriggeringStopWithSavepointWithDrain() throws Exception {
    SourceOperatorFactory<Integer> sourceOperatorFactory = new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 2), WatermarkStrategy.noWatermarks());
    CompletableFuture<Boolean> checkpointCompleted = new CompletableFuture<>();
    CheckpointResponder checkpointResponder = new TestCheckpointResponder() {

        @Override
        public void acknowledgeCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
            super.acknowledgeCheckpoint(jobID, executionAttemptID, checkpointId, checkpointMetrics, subtaskState);
            checkpointCompleted.complete(null);
        }
    };
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(new LifeCycleMonitorMultipleInputOperatorFactory()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).setCheckpointResponder(checkpointResponder).build()) {
        CompletableFuture<Boolean> triggerResult = testHarness.streamTask.triggerCheckpointAsync(new CheckpointMetaData(2, 2), CheckpointOptions.alignedNoTimeout(SavepointType.terminate(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()));
        checkpointCompleted.whenComplete((ignored, exception) -> testHarness.streamTask.notifyCheckpointCompleteAsync(2));
        testHarness.waitForTaskCompletion();
        testHarness.finishProcessing();
        assertTrue(triggerResult.isDone());
        assertTrue(triggerResult.get());
        assertTrue(checkpointCompleted.isDone());
    }
}

Example 2 with SourceOperatorFactory

use of org.apache.flink.streaming.api.operators.SourceOperatorFactory in project flink by apache.

the class MultipleInputStreamTaskTest method testCopyForObjectReuse.

@Test
public void testCopyForObjectReuse() throws Exception {
    SharedReference<List<Integer>> copiedElementsRef = sharedObjects.add(new ArrayList<>());
    CopyProxySerializer proxySerializer = new CopyProxySerializer(copiedElementsRef);
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.BOUNDED, 1), WatermarkStrategy.noWatermarks()), proxySerializer).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory(3)).build()) {
        addSourceRecords(testHarness, 1, 42, 43);
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
        if (objectReuse) {
            assertTrue(copiedElementsRef.get().isEmpty());
        } else {
            assertThat(copiedElementsRef.get(), containsInAnyOrder(42, 43));
        }
    }
}

Example 3 with SourceOperatorFactory

use of org.apache.flink.streaming.api.operators.SourceOperatorFactory in project flink by apache.

the class MultipleInputStreamTaskTest method testMetrics.

/**
 * With chained sources, task's and main operator's number of input records are two different
 * things. The first one should take into account only records comming in from the network,
 * ignoring records produced inside the task itself (like via a chained source). Main operator
 * should on the other hand report all records from all of the inputs (regardless if it's a
 * network or chained input).
 */
@Test
public void testMetrics() throws Exception {
    HashMap<String, OperatorMetricGroup> operatorMetrics = new HashMap<>();
    TaskMetricGroup taskMetricGroup = new UnregisteredMetricGroups.UnregisteredTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID operatorID, String name) {
            InternalOperatorMetricGroup operatorMetricGroup = super.getOrAddOperator(operatorID, name);
            operatorMetrics.put(name, operatorMetricGroup);
            return operatorMetricGroup;
        }
    };
    String mainOperatorName = "MainOperator";
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new LifeCycleTrackingMockSource(Boundedness.BOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(3)).name(mainOperatorName).chain(new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).chain(new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).chain(new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish().setTaskMetricGroup(taskMetricGroup).build()) {
        assertTrue(operatorMetrics.containsKey(mainOperatorName));
        OperatorMetricGroup mainOperatorMetrics = operatorMetrics.get(mainOperatorName);
        Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter();
        Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();
        int numRecords1 = 5;
        int numRecords2 = 3;
        int numRecords3 = 2;
        // end prematurely
        for (int x = 0; x < numRecords2; x++) {
            addSourceRecords(testHarness, 1, 42);
        }
        for (int x = 0; x < numRecords1; x++) {
            testHarness.processElement(new StreamRecord<>("hello"), 0, 0);
        }
        for (int x = 0; x < numRecords3; x++) {
            testHarness.processElement(new StreamRecord<>("hello"), 1, 0);
        }
        int networkRecordsIn = numRecords1 + numRecords3;
        int mainOperatorRecordsIn = networkRecordsIn + numRecords2;
        int totalRecordsOut = mainOperatorRecordsIn * 2 * 2 * // there are three operators duplicating the records
        2;
        assertEquals(mainOperatorRecordsIn, mainOperatorMetrics.getIOMetricGroup().getNumRecordsInCounter().getCount());
        assertEquals(networkRecordsIn, numRecordsInCounter.getCount());
        assertEquals(totalRecordsOut, numRecordsOutCounter.getCount());
        testHarness.waitForTaskCompletion();
    }
}

Example 4 with SourceOperatorFactory

use of org.apache.flink.streaming.api.operators.SourceOperatorFactory in project flink by apache.

the class SourceOperatorLatencyMetricsTest method testLatencyMarkEmission.

private void testLatencyMarkEmission(boolean shouldExpectLatencyMarkers, Configuration taskManagerConfig, ExecutionConfig executionConfig) throws Exception {
    try (SourceOperatorTestHarness testHarness = new SourceOperatorTestHarness(new SourceOperatorFactory(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), new MockEnvironmentBuilder().setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(taskManagerConfig)).setExecutionConfig(executionConfig).build())) {
        testHarness.open();
        testHarness.setup();
        for (long processingTime = 0; processingTime <= MAX_PROCESSING_TIME; processingTime++) {
            testHarness.getProcessingTimeService().setCurrentTime(processingTime);
            testHarness.emitNext();
        }
        List<LatencyMarker> expectedOutput = new ArrayList<>();
        if (!shouldExpectLatencyMarkers) {
            assertTrue(testHarness.getOutput().isEmpty());
        } else {
            expectedOutput.add(new LatencyMarker(1, testHarness.getOperator().getOperatorID(), 0));
            for (long markedTime = LATENCY_MARK_INTERVAL; markedTime <= MAX_PROCESSING_TIME; markedTime += LATENCY_MARK_INTERVAL) {
                expectedOutput.add(new LatencyMarker(markedTime, testHarness.getOperator().getOperatorID(), 0));
            }
            assertThat((Collection<Object>) testHarness.getOutput(), contains(expectedOutput.toArray()));
        }
    }
}

Example 5 with SourceOperatorFactory

use of org.apache.flink.streaming.api.operators.SourceOperatorFactory in project flink by apache.

the class SourceTransformationTranslator method translateInternal.

private Collection<Integer> translateInternal(final SourceTransformation<OUT, SplitT, EnumChkT> transformation, final Context context, boolean emitProgressiveWatermarks) {
    checkNotNull(transformation);
    checkNotNull(context);
    final StreamGraph streamGraph = context.getStreamGraph();
    final String slotSharingGroup = context.getSlotSharingGroup();
    final int transformationId = transformation.getId();
    final ExecutionConfig executionConfig = streamGraph.getExecutionConfig();
    SourceOperatorFactory<OUT> operatorFactory = new SourceOperatorFactory<>(transformation.getSource(), transformation.getWatermarkStrategy(), emitProgressiveWatermarks);
    operatorFactory.setChainingStrategy(transformation.getChainingStrategy());
    streamGraph.addSource(transformationId, slotSharingGroup, transformation.getCoLocationGroupKey(), operatorFactory, null, transformation.getOutputType(), "Source: " + transformation.getName());
    final int parallelism = transformation.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ? transformation.getParallelism() : executionConfig.getParallelism();
    streamGraph.setParallelism(transformationId, parallelism);
    streamGraph.setMaxParallelism(transformationId, transformation.getMaxParallelism());
    return Collections.singleton(transformationId);
}