use of org.apache.flink.streaming.runtime.watermarkstatus.StatusWatermarkValve in project flink by apache.
the class StreamMultipleInputProcessorFactory method create.
@SuppressWarnings({ "unchecked", "rawtypes" })
public static StreamMultipleInputProcessor create(TaskInvokable ownerTask, CheckpointedInputGate[] checkpointedInputGates, StreamConfig.InputConfig[] configuredInputs, IOManager ioManager, MemoryManager memoryManager, TaskIOMetricGroup ioMetricGroup, Counter mainOperatorRecordsIn, MultipleInputStreamOperator<?> mainOperator, WatermarkGauge[] inputWatermarkGauges, StreamConfig streamConfig, Configuration taskManagerConfig, Configuration jobConfig, ExecutionConfig executionConfig, ClassLoader userClassloader, OperatorChain<?, ?> operatorChain, InflightDataRescalingDescriptor inflightDataRescalingDescriptor, Function<Integer, StreamPartitioner<?>> gatePartitioners, TaskInfo taskInfo) {
checkNotNull(operatorChain);
List<Input> operatorInputs = mainOperator.getInputs();
int inputsCount = operatorInputs.size();
StreamOneInputProcessor<?>[] inputProcessors = new StreamOneInputProcessor[inputsCount];
Counter networkRecordsIn = new SimpleCounter();
ioMetricGroup.reuseRecordsInputCounter(networkRecordsIn);
checkState(configuredInputs.length == inputsCount, "Number of configured inputs in StreamConfig [%s] doesn't match the main operator's number of inputs [%s]", configuredInputs.length, inputsCount);
StreamTaskInput[] inputs = new StreamTaskInput[inputsCount];
for (int i = 0; i < inputsCount; i++) {
StreamConfig.InputConfig configuredInput = configuredInputs[i];
if (configuredInput instanceof StreamConfig.NetworkInputConfig) {
StreamConfig.NetworkInputConfig networkInput = (StreamConfig.NetworkInputConfig) configuredInput;
inputs[i] = StreamTaskNetworkInputFactory.create(checkpointedInputGates[networkInput.getInputGateIndex()], networkInput.getTypeSerializer(), ioManager, new StatusWatermarkValve(checkpointedInputGates[networkInput.getInputGateIndex()].getNumberOfInputChannels()), i, inflightDataRescalingDescriptor, gatePartitioners, taskInfo);
} else if (configuredInput instanceof StreamConfig.SourceInputConfig) {
StreamConfig.SourceInputConfig sourceInput = (StreamConfig.SourceInputConfig) configuredInput;
inputs[i] = operatorChain.getSourceTaskInput(sourceInput);
} else {
throw new UnsupportedOperationException("Unknown input type: " + configuredInput);
}
}
InputSelectable inputSelectable = mainOperator instanceof InputSelectable ? (InputSelectable) mainOperator : null;
StreamConfig.InputConfig[] inputConfigs = streamConfig.getInputs(userClassloader);
boolean anyRequiresSorting = Arrays.stream(inputConfigs).anyMatch(StreamConfig::requiresSorting);
if (anyRequiresSorting) {
if (inputSelectable != null) {
throw new IllegalStateException("The InputSelectable interface is not supported with sorting inputs");
}
StreamTaskInput[] sortingInputs = IntStream.range(0, inputsCount).filter(idx -> requiresSorting(inputConfigs[idx])).mapToObj(idx -> inputs[idx]).toArray(StreamTaskInput[]::new);
KeySelector[] sortingInputKeySelectors = IntStream.range(0, inputsCount).filter(idx -> requiresSorting(inputConfigs[idx])).mapToObj(idx -> streamConfig.getStatePartitioner(idx, userClassloader)).toArray(KeySelector[]::new);
TypeSerializer[] sortingInputKeySerializers = IntStream.range(0, inputsCount).filter(idx -> requiresSorting(inputConfigs[idx])).mapToObj(idx -> streamConfig.getTypeSerializerIn(idx, userClassloader)).toArray(TypeSerializer[]::new);
StreamTaskInput[] passThroughInputs = IntStream.range(0, inputsCount).filter(idx -> !requiresSorting(inputConfigs[idx])).mapToObj(idx -> inputs[idx]).toArray(StreamTaskInput[]::new);
SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(ownerTask, sortingInputs, sortingInputKeySelectors, sortingInputKeySerializers, streamConfig.getStateKeySerializer(userClassloader), passThroughInputs, memoryManager, ioManager, executionConfig.isObjectReuseEnabled(), streamConfig.getManagedMemoryFractionOperatorUseCaseOfSlot(ManagedMemoryUseCase.OPERATOR, taskManagerConfig, userClassloader), jobConfig, executionConfig);
StreamTaskInput<?>[] sortedInputs = selectableSortingInputs.getSortedInputs();
StreamTaskInput<?>[] passedThroughInputs = selectableSortingInputs.getPassThroughInputs();
int sortedIndex = 0;
int passThroughIndex = 0;
for (int i = 0; i < inputs.length; i++) {
if (requiresSorting(inputConfigs[i])) {
inputs[i] = sortedInputs[sortedIndex];
sortedIndex++;
} else {
inputs[i] = passedThroughInputs[passThroughIndex];
passThroughIndex++;
}
}
inputSelectable = selectableSortingInputs.getInputSelectable();
}
for (int i = 0; i < inputsCount; i++) {
StreamConfig.InputConfig configuredInput = configuredInputs[i];
if (configuredInput instanceof StreamConfig.NetworkInputConfig) {
StreamTaskNetworkOutput dataOutput = new StreamTaskNetworkOutput<>(operatorChain.getFinishedOnRestoreInputOrDefault(operatorInputs.get(i)), inputWatermarkGauges[i], mainOperatorRecordsIn, networkRecordsIn);
inputProcessors[i] = new StreamOneInputProcessor(inputs[i], dataOutput, operatorChain);
} else if (configuredInput instanceof StreamConfig.SourceInputConfig) {
StreamConfig.SourceInputConfig sourceInput = (StreamConfig.SourceInputConfig) configuredInput;
OperatorChain.ChainedSource chainedSource = operatorChain.getChainedSource(sourceInput);
inputProcessors[i] = new StreamOneInputProcessor(inputs[i], new StreamTaskSourceOutput(chainedSource.getSourceOutput(), inputWatermarkGauges[i], chainedSource.getSourceTaskInput().getOperator().getSourceMetricGroup()), operatorChain);
} else {
throw new UnsupportedOperationException("Unknown input type: " + configuredInput);
}
}
return new StreamMultipleInputProcessor(new MultipleInputSelectionHandler(inputSelectable, inputsCount), inputProcessors);
}
use of org.apache.flink.streaming.runtime.watermarkstatus.StatusWatermarkValve in project flink by apache.
the class OneInputStreamTask method createTaskInput.
private StreamTaskInput<IN> createTaskInput(CheckpointedInputGate inputGate) {
int numberOfInputChannels = inputGate.getNumberOfInputChannels();
StatusWatermarkValve statusWatermarkValve = new StatusWatermarkValve(numberOfInputChannels);
TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader());
return StreamTaskNetworkInputFactory.create(inputGate, inSerializer, getEnvironment().getIOManager(), statusWatermarkValve, 0, getEnvironment().getTaskStateManager().getInputRescalingDescriptor(), gateIndex -> configuration.getInPhysicalEdges(getUserCodeClassLoader()).get(gateIndex).getPartitioner(), getEnvironment().getTaskInfo());
}
use of org.apache.flink.streaming.runtime.watermarkstatus.StatusWatermarkValve in project flink by apache.
the class StreamTaskNetworkInputTest method testSnapshotAfterEndOfPartition.
@Test
public void testSnapshotAfterEndOfPartition() throws Exception {
int numInputChannels = 1;
int channelId = 0;
int checkpointId = 0;
VerifyRecordsDataOutput<Long> output = new VerifyRecordsDataOutput<>();
LongSerializer inSerializer = LongSerializer.INSTANCE;
StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
StreamTaskInput<Long> input = new StreamTaskNetworkInput<>(new CheckpointedInputGate(inputGate.getInputGate(), SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", new DummyCheckpointInvokable(), SystemClock.getInstance(), false, inputGate.getInputGate()), new SyncMailboxExecutor()), inSerializer, ioManager, new StatusWatermarkValve(numInputChannels), 0);
inputGate.sendEvent(new CheckpointBarrier(checkpointId, 0L, CheckpointOptions.forCheckpointWithDefaultLocation().toUnaligned()), channelId);
inputGate.sendElement(new StreamRecord<>(42L), channelId);
assertHasNextElement(input, output);
assertHasNextElement(input, output);
assertEquals(1, output.getNumberOfEmittedRecords());
// send EndOfPartitionEvent and ensure that deserializer has been released
inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelId);
input.emitNext(output);
// now snapshot all inflight buffers
CompletableFuture<Void> completableFuture = input.prepareSnapshot(ChannelStateWriter.NO_OP, checkpointId);
completableFuture.join();
}
use of org.apache.flink.streaming.runtime.watermarkstatus.StatusWatermarkValve in project flink by apache.
the class StreamTwoInputProcessorFactory method create.
public static <IN1, IN2> StreamMultipleInputProcessor create(TaskInvokable ownerTask, CheckpointedInputGate[] checkpointedInputGates, IOManager ioManager, MemoryManager memoryManager, TaskIOMetricGroup taskIOMetricGroup, TwoInputStreamOperator<IN1, IN2, ?> streamOperator, WatermarkGauge input1WatermarkGauge, WatermarkGauge input2WatermarkGauge, OperatorChain<?, ?> operatorChain, StreamConfig streamConfig, Configuration taskManagerConfig, Configuration jobConfig, ExecutionConfig executionConfig, ClassLoader userClassloader, Counter numRecordsIn, InflightDataRescalingDescriptor inflightDataRescalingDescriptor, Function<Integer, StreamPartitioner<?>> gatePartitioners, TaskInfo taskInfo) {
checkNotNull(operatorChain);
taskIOMetricGroup.reuseRecordsInputCounter(numRecordsIn);
TypeSerializer<IN1> typeSerializer1 = streamConfig.getTypeSerializerIn(0, userClassloader);
StreamTaskInput<IN1> input1 = StreamTaskNetworkInputFactory.create(checkpointedInputGates[0], typeSerializer1, ioManager, new StatusWatermarkValve(checkpointedInputGates[0].getNumberOfInputChannels()), 0, inflightDataRescalingDescriptor, gatePartitioners, taskInfo);
TypeSerializer<IN2> typeSerializer2 = streamConfig.getTypeSerializerIn(1, userClassloader);
StreamTaskInput<IN2> input2 = StreamTaskNetworkInputFactory.create(checkpointedInputGates[1], typeSerializer2, ioManager, new StatusWatermarkValve(checkpointedInputGates[1].getNumberOfInputChannels()), 1, inflightDataRescalingDescriptor, gatePartitioners, taskInfo);
InputSelectable inputSelectable = streamOperator instanceof InputSelectable ? (InputSelectable) streamOperator : null;
// this is a bit verbose because we're manually handling input1 and input2
// TODO: extract method
StreamConfig.InputConfig[] inputConfigs = streamConfig.getInputs(userClassloader);
boolean input1IsSorted = requiresSorting(inputConfigs[0]);
boolean input2IsSorted = requiresSorting(inputConfigs[1]);
if (input1IsSorted || input2IsSorted) {
if (inputSelectable != null) {
throw new IllegalStateException("The InputSelectable interface is not supported with sorting inputs");
}
List<StreamTaskInput<?>> sortedTaskInputs = new ArrayList<>();
List<KeySelector<?, ?>> keySelectors = new ArrayList<>();
List<StreamTaskInput<?>> passThroughTaskInputs = new ArrayList<>();
if (input1IsSorted) {
sortedTaskInputs.add(input1);
keySelectors.add(streamConfig.getStatePartitioner(0, userClassloader));
} else {
passThroughTaskInputs.add(input1);
}
if (input2IsSorted) {
sortedTaskInputs.add(input2);
keySelectors.add(streamConfig.getStatePartitioner(1, userClassloader));
} else {
passThroughTaskInputs.add(input2);
}
@SuppressWarnings("unchecked") SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(ownerTask, sortedTaskInputs.toArray(new StreamTaskInput[0]), keySelectors.toArray(new KeySelector[0]), new TypeSerializer[] { typeSerializer1, typeSerializer2 }, streamConfig.getStateKeySerializer(userClassloader), passThroughTaskInputs.toArray(new StreamTaskInput[0]), memoryManager, ioManager, executionConfig.isObjectReuseEnabled(), streamConfig.getManagedMemoryFractionOperatorUseCaseOfSlot(ManagedMemoryUseCase.OPERATOR, taskManagerConfig, userClassloader), jobConfig, executionConfig);
inputSelectable = selectableSortingInputs.getInputSelectable();
StreamTaskInput<?>[] sortedInputs = selectableSortingInputs.getSortedInputs();
StreamTaskInput<?>[] passThroughInputs = selectableSortingInputs.getPassThroughInputs();
if (input1IsSorted) {
input1 = toTypedInput(sortedInputs[0]);
} else {
input1 = toTypedInput(passThroughInputs[0]);
}
if (input2IsSorted) {
input2 = toTypedInput(sortedInputs[sortedInputs.length - 1]);
} else {
input2 = toTypedInput(passThroughInputs[passThroughInputs.length - 1]);
}
}
@Nullable FinishedOnRestoreWatermarkBypass watermarkBypass = operatorChain.isTaskDeployedAsFinished() ? new FinishedOnRestoreWatermarkBypass(operatorChain.getStreamOutputs()) : null;
StreamTaskNetworkOutput<IN1> output1 = new StreamTaskNetworkOutput<>(streamOperator, record -> processRecord1(record, streamOperator), input1WatermarkGauge, 0, numRecordsIn, watermarkBypass);
StreamOneInputProcessor<IN1> processor1 = new StreamOneInputProcessor<>(input1, output1, operatorChain);
StreamTaskNetworkOutput<IN2> output2 = new StreamTaskNetworkOutput<>(streamOperator, record -> processRecord2(record, streamOperator), input2WatermarkGauge, 1, numRecordsIn, watermarkBypass);
StreamOneInputProcessor<IN2> processor2 = new StreamOneInputProcessor<>(input2, output2, operatorChain);
return new StreamMultipleInputProcessor(new MultipleInputSelectionHandler(inputSelectable, 2), new StreamOneInputProcessor[] { processor1, processor2 });
}
Aggregations