Examples with TaskInvokable org.apache.flink.runtime.jobgraph.tasks.TaskInvokable used on opensource projects

Example 6 with TaskInvokable

use of org.apache.flink.runtime.jobgraph.tasks.TaskInvokable in project flink by apache.

the class StreamingJobGraphGenerator method setVertexConfig.

private void setVertexConfig(Integer vertexID, StreamConfig config, List<StreamEdge> chainableOutputs, List<StreamEdge> nonChainableOutputs, Map<Integer, ChainedSourceInfo> chainedSources) {
    tryConvertPartitionerForDynamicGraph(chainableOutputs, nonChainableOutputs);
    StreamNode vertex = streamGraph.getStreamNode(vertexID);
    config.setVertexID(vertexID);
    // build the inputs as a combination of source and network inputs
    final List<StreamEdge> inEdges = vertex.getInEdges();
    final TypeSerializer<?>[] inputSerializers = vertex.getTypeSerializersIn();
    final StreamConfig.InputConfig[] inputConfigs = new StreamConfig.InputConfig[inputSerializers.length];
    int inputGateCount = 0;
    for (final StreamEdge inEdge : inEdges) {
        final ChainedSourceInfo chainedSource = chainedSources.get(inEdge.getSourceId());
        final int inputIndex = inEdge.getTypeNumber() == 0 ? // single input operator
        0 : // in case of 2 or more inputs
        inEdge.getTypeNumber() - 1;
        if (chainedSource != null) {
            // chained source is the input
            if (inputConfigs[inputIndex] != null) {
                throw new IllegalStateException("Trying to union a chained source with another input.");
            }
            inputConfigs[inputIndex] = chainedSource.getInputConfig();
            chainedConfigs.computeIfAbsent(vertexID, (key) -> new HashMap<>()).put(inEdge.getSourceId(), chainedSource.getOperatorConfig());
        } else {
            // that is union-ed into the same input
            if (inputConfigs[inputIndex] == null) {
                // PASS_THROUGH is a sensible default for streaming jobs. Only for BATCH
                // execution can we have sorted inputs
                StreamConfig.InputRequirement inputRequirement = vertex.getInputRequirements().getOrDefault(inputIndex, StreamConfig.InputRequirement.PASS_THROUGH);
                inputConfigs[inputIndex] = new StreamConfig.NetworkInputConfig(inputSerializers[inputIndex], inputGateCount++, inputRequirement);
            }
        }
    }
    config.setInputs(inputConfigs);
    config.setTypeSerializerOut(vertex.getTypeSerializerOut());
    // iterate edges, find sideOutput edges create and save serializers for each outputTag type
    for (StreamEdge edge : chainableOutputs) {
        if (edge.getOutputTag() != null) {
            config.setTypeSerializerSideOut(edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()));
        }
    }
    for (StreamEdge edge : nonChainableOutputs) {
        if (edge.getOutputTag() != null) {
            config.setTypeSerializerSideOut(edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()));
        }
    }
    config.setStreamOperatorFactory(vertex.getOperatorFactory());
    config.setNumberOfOutputs(nonChainableOutputs.size());
    config.setNonChainedOutputs(nonChainableOutputs);
    config.setChainedOutputs(chainableOutputs);
    config.setTimeCharacteristic(streamGraph.getTimeCharacteristic());
    final CheckpointConfig checkpointCfg = streamGraph.getCheckpointConfig();
    config.setStateBackend(streamGraph.getStateBackend());
    config.setChangelogStateBackendEnabled(streamGraph.isChangelogStateBackendEnabled());
    config.setCheckpointStorage(streamGraph.getCheckpointStorage());
    config.setSavepointDir(streamGraph.getSavepointDirectory());
    config.setGraphContainingLoops(streamGraph.isIterative());
    config.setTimerServiceProvider(streamGraph.getTimerServiceProvider());
    config.setCheckpointingEnabled(checkpointCfg.isCheckpointingEnabled());
    config.getConfiguration().set(ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, streamGraph.isEnableCheckpointsAfterTasksFinish());
    config.setCheckpointMode(getCheckpointingMode(checkpointCfg));
    config.setUnalignedCheckpointsEnabled(checkpointCfg.isUnalignedCheckpointsEnabled());
    config.setAlignedCheckpointTimeout(checkpointCfg.getAlignedCheckpointTimeout());
    for (int i = 0; i < vertex.getStatePartitioners().length; i++) {
        config.setStatePartitioner(i, vertex.getStatePartitioners()[i]);
    }
    config.setStateKeySerializer(vertex.getStateKeySerializer());
    Class<? extends TaskInvokable> vertexClass = vertex.getJobVertexClass();
    if (vertexClass.equals(StreamIterationHead.class) || vertexClass.equals(StreamIterationTail.class)) {
        config.setIterationId(streamGraph.getBrokerID(vertexID));
        config.setIterationWaitTime(streamGraph.getLoopTimeout(vertexID));
    }
    vertexConfigs.put(vertexID, config);
}

Example 7 with TaskInvokable

use of org.apache.flink.runtime.jobgraph.tasks.TaskInvokable in project flink by apache.

the class MultiInputSortingDataInput method wrapInputs.

public static <K> SelectableSortingInputs wrapInputs(TaskInvokable containingTask, StreamTaskInput<Object>[] sortingInputs, KeySelector<Object, K>[] keySelectors, TypeSerializer<Object>[] inputSerializers, TypeSerializer<K> keySerializer, StreamTaskInput<Object>[] passThroughInputs, MemoryManager memoryManager, IOManager ioManager, boolean objectReuse, double managedMemoryFraction, Configuration jobConfiguration, ExecutionConfig executionConfig) {
    int keyLength = keySerializer.getLength();
    final TypeComparator<Tuple2<byte[], StreamRecord<Object>>> comparator;
    DataOutputSerializer dataOutputSerializer;
    if (keyLength > 0) {
        dataOutputSerializer = new DataOutputSerializer(keyLength);
        comparator = new FixedLengthByteKeyComparator<>(keyLength);
    } else {
        dataOutputSerializer = new DataOutputSerializer(64);
        comparator = new VariableLengthByteKeyComparator<>();
    }
    List<Integer> passThroughInputIndices = Arrays.stream(passThroughInputs).map(StreamTaskInput::getInputIndex).collect(Collectors.toList());
    int numberOfInputs = sortingInputs.length + passThroughInputs.length;
    CommonContext commonContext = new CommonContext(sortingInputs);
    InputSelector inputSelector = new InputSelector(commonContext, numberOfInputs, passThroughInputIndices);
    StreamTaskInput<?>[] wrappedSortingInputs = IntStream.range(0, sortingInputs.length).mapToObj(idx -> {
        try {
            KeyAndValueSerializer<Object> keyAndValueSerializer = new KeyAndValueSerializer<>(inputSerializers[idx], keyLength);
            return new MultiInputSortingDataInput<>(commonContext, sortingInputs[idx], sortingInputs[idx].getInputIndex(), ExternalSorter.newBuilder(memoryManager, containingTask, keyAndValueSerializer, comparator, executionConfig).memoryFraction(managedMemoryFraction / numberOfInputs).enableSpilling(ioManager, jobConfiguration.get(AlgorithmOptions.SORT_SPILLING_THRESHOLD)).maxNumFileHandles(jobConfiguration.get(AlgorithmOptions.SPILLING_MAX_FAN) / numberOfInputs).objectReuse(objectReuse).largeRecords(true).build(), keySelectors[idx], keySerializer, dataOutputSerializer);
        } catch (MemoryAllocationException e) {
            throw new RuntimeException();
        }
    }).toArray(StreamTaskInput[]::new);
    StreamTaskInput<?>[] wrappedPassThroughInputs = Arrays.stream(passThroughInputs).map(input -> new ObservableStreamTaskInput<>(input, inputSelector)).toArray(StreamTaskInput[]::new);
    return new SelectableSortingInputs(wrappedSortingInputs, wrappedPassThroughInputs, inputSelector);
}