Examples with ManagedMemoryUseCase org.apache.flink.core.memory.ManagedMemoryUseCase used on opensource projects

Example 1 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class PythonOperatorChainingOptimizer method createChainedTransformation.

private static Transformation<?> createChainedTransformation(Transformation<?> upTransform, Transformation<?> downTransform) {
    final AbstractDataStreamPythonFunctionOperator<?> upOperator = (AbstractDataStreamPythonFunctionOperator<?>) ((SimpleOperatorFactory<?>) getOperatorFactory(upTransform)).getOperator();
    final PythonProcessOperator<?, ?> downOperator = (PythonProcessOperator<?, ?>) ((SimpleOperatorFactory<?>) getOperatorFactory(downTransform)).getOperator();
    final DataStreamPythonFunctionInfo upPythonFunctionInfo = upOperator.getPythonFunctionInfo().copy();
    final DataStreamPythonFunctionInfo downPythonFunctionInfo = downOperator.getPythonFunctionInfo().copy();
    DataStreamPythonFunctionInfo headPythonFunctionInfoOfDownOperator = downPythonFunctionInfo;
    while (headPythonFunctionInfoOfDownOperator.getInputs().length != 0) {
        headPythonFunctionInfoOfDownOperator = (DataStreamPythonFunctionInfo) headPythonFunctionInfoOfDownOperator.getInputs()[0];
    }
    headPythonFunctionInfoOfDownOperator.setInputs(new DataStreamPythonFunctionInfo[] { upPythonFunctionInfo });
    final AbstractDataStreamPythonFunctionOperator<?> chainedOperator = upOperator.copy(downPythonFunctionInfo, downOperator.getProducedType());
    // set partition custom flag
    chainedOperator.setContainsPartitionCustom(downOperator.containsPartitionCustom() || upOperator.containsPartitionCustom());
    PhysicalTransformation<?> chainedTransformation;
    if (upOperator instanceof AbstractOneInputPythonFunctionOperator) {
        chainedTransformation = new OneInputTransformation(upTransform.getInputs().get(0), upTransform.getName() + ", " + downTransform.getName(), (OneInputStreamOperator<?, ?>) chainedOperator, downTransform.getOutputType(), upTransform.getParallelism());
        ((OneInputTransformation<?, ?>) chainedTransformation).setStateKeySelector(((OneInputTransformation) upTransform).getStateKeySelector());
        ((OneInputTransformation<?, ?>) chainedTransformation).setStateKeyType(((OneInputTransformation<?, ?>) upTransform).getStateKeyType());
    } else {
        chainedTransformation = new TwoInputTransformation(upTransform.getInputs().get(0), upTransform.getInputs().get(1), upTransform.getName() + ", " + downTransform.getName(), (TwoInputStreamOperator<?, ?, ?>) chainedOperator, downTransform.getOutputType(), upTransform.getParallelism());
        ((TwoInputTransformation<?, ?, ?>) chainedTransformation).setStateKeySelectors(((TwoInputTransformation) upTransform).getStateKeySelector1(), ((TwoInputTransformation) upTransform).getStateKeySelector2());
        ((TwoInputTransformation<?, ?, ?>) chainedTransformation).setStateKeyType(((TwoInputTransformation<?, ?, ?>) upTransform).getStateKeyType());
    }
    chainedTransformation.setUid(upTransform.getUid());
    if (upTransform.getUserProvidedNodeHash() != null) {
        chainedTransformation.setUidHash(upTransform.getUserProvidedNodeHash());
    }
    for (ManagedMemoryUseCase useCase : upTransform.getManagedMemorySlotScopeUseCases()) {
        chainedTransformation.declareManagedMemoryUseCaseAtSlotScope(useCase);
    }
    for (ManagedMemoryUseCase useCase : downTransform.getManagedMemorySlotScopeUseCases()) {
        chainedTransformation.declareManagedMemoryUseCaseAtSlotScope(useCase);
    }
    for (Map.Entry<ManagedMemoryUseCase, Integer> useCase : upTransform.getManagedMemoryOperatorScopeUseCaseWeights().entrySet()) {
        chainedTransformation.declareManagedMemoryUseCaseAtOperatorScope(useCase.getKey(), useCase.getValue());
    }
    for (Map.Entry<ManagedMemoryUseCase, Integer> useCase : downTransform.getManagedMemoryOperatorScopeUseCaseWeights().entrySet()) {
        chainedTransformation.declareManagedMemoryUseCaseAtOperatorScope(useCase.getKey(), useCase.getValue() + chainedTransformation.getManagedMemoryOperatorScopeUseCaseWeights().getOrDefault(useCase.getKey(), 0));
    }
    chainedTransformation.setBufferTimeout(Math.min(upTransform.getBufferTimeout(), downTransform.getBufferTimeout()));
    if (upTransform.getMaxParallelism() > 0) {
        chainedTransformation.setMaxParallelism(upTransform.getMaxParallelism());
    }
    chainedTransformation.setChainingStrategy(getOperatorFactory(upTransform).getChainingStrategy());
    chainedTransformation.setCoLocationGroupKey(upTransform.getCoLocationGroupKey());
    chainedTransformation.setResources(upTransform.getMinResources().merge(downTransform.getMinResources()), upTransform.getPreferredResources().merge(downTransform.getPreferredResources()));
    if (upTransform.getSlotSharingGroup().isPresent()) {
        chainedTransformation.setSlotSharingGroup(upTransform.getSlotSharingGroup().get());
    }
    if (upTransform.getDescription() != null && downTransform.getDescription() != null) {
        chainedTransformation.setDescription(upTransform.getDescription() + ", " + downTransform.getDescription());
    } else if (upTransform.getDescription() != null) {
        chainedTransformation.setDescription(upTransform.getDescription());
    } else if (downTransform.getDescription() != null) {
        chainedTransformation.setDescription(downTransform.getDescription());
    }
    return chainedTransformation;
}

Example 2 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class BatchExecutionUtils method applyBatchExecutionSettings.

static void applyBatchExecutionSettings(int transformationId, TransformationTranslator.Context context, StreamConfig.InputRequirement... inputRequirements) {
    StreamNode node = context.getStreamGraph().getStreamNode(transformationId);
    boolean sortInputs = context.getGraphGeneratorConfig().get(ExecutionOptions.SORT_INPUTS);
    boolean isInputSelectable = isInputSelectable(node);
    adjustChainingStrategy(node);
    checkState(!isInputSelectable || !sortInputs, "Batch state backend and sorting inputs are not supported in graphs with an InputSelectable operator.");
    if (sortInputs) {
        LOG.debug("Applying sorting/pass-through input requirements for operator {}.", node);
        for (int i = 0; i < inputRequirements.length; i++) {
            node.addInputRequirement(i, inputRequirements[i]);
        }
        Map<ManagedMemoryUseCase, Integer> operatorScopeUseCaseWeights = new HashMap<>();
        operatorScopeUseCaseWeights.put(ManagedMemoryUseCase.OPERATOR, deriveMemoryWeight(context.getGraphGeneratorConfig()));
        node.setManagedMemoryUseCaseWeights(operatorScopeUseCaseWeights, Collections.emptySet());
    }
}

Example 3 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class StreamingJobGraphGeneratorTest method createJobGraphForManagedMemoryFractionTest.

private JobGraph createJobGraphForManagedMemoryFractionTest(final List<ResourceSpec> resourceSpecs, final List<Map<ManagedMemoryUseCase, Integer>> operatorScopeUseCaseWeights, final List<Set<ManagedMemoryUseCase>> slotScopeUseCases) throws Exception {
    final Method opMethod = getSetResourcesMethodAndSetAccessible(SingleOutputStreamOperator.class);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> source = env.addSource(new ParallelSourceFunction<Integer>() {

        @Override
        public void run(SourceContext<Integer> ctx) {
        }

        @Override
        public void cancel() {
        }
    });
    opMethod.invoke(source, resourceSpecs.get(0));
    // CHAIN(source -> map1) in default slot sharing group
    final DataStream<Integer> map1 = source.map((MapFunction<Integer, Integer>) value -> value);
    opMethod.invoke(map1, resourceSpecs.get(1));
    // CHAIN(map2) in default slot sharing group
    final DataStream<Integer> map2 = map1.rebalance().map((MapFunction<Integer, Integer>) value -> value);
    opMethod.invoke(map2, resourceSpecs.get(2));
    // CHAIN(map3) in test slot sharing group
    final DataStream<Integer> map3 = map2.rebalance().map(value -> value).slotSharingGroup("test");
    opMethod.invoke(map3, resourceSpecs.get(3));
    declareManagedMemoryUseCaseForTranformation(source.getTransformation(), operatorScopeUseCaseWeights.get(0), slotScopeUseCases.get(0));
    declareManagedMemoryUseCaseForTranformation(map1.getTransformation(), operatorScopeUseCaseWeights.get(1), slotScopeUseCases.get(1));
    declareManagedMemoryUseCaseForTranformation(map2.getTransformation(), operatorScopeUseCaseWeights.get(2), slotScopeUseCases.get(2));
    declareManagedMemoryUseCaseForTranformation(map3.getTransformation(), operatorScopeUseCaseWeights.get(3), slotScopeUseCases.get(3));
    return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 4 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class StreamGraphGeneratorBatchExecutionTest method testManagedMemoryWeights.

@Test
public void testManagedMemoryWeights() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    SingleOutputStreamOperator<Integer> process = env.fromElements(1, 2).keyBy(Integer::intValue).process(DUMMY_PROCESS_FUNCTION);
    DataStreamSink<Integer> sink = process.addSink(new DiscardingSink<>());
    StreamGraph graph = getStreamGraphInBatchMode(sink);
    StreamNode processNode = graph.getStreamNode(process.getId());
    final Map<ManagedMemoryUseCase, Integer> expectedOperatorWeights = new HashMap<>();
    expectedOperatorWeights.put(ManagedMemoryUseCase.OPERATOR, ExecutionOptions.SORTED_INPUTS_MEMORY.defaultValue().getMebiBytes());
    assertThat(processNode.getManagedMemoryOperatorScopeUseCaseWeights(), equalTo(expectedOperatorWeights));
    assertThat(processNode.getManagedMemorySlotScopeUseCases(), equalTo(Collections.singleton(ManagedMemoryUseCase.STATE_BACKEND)));
}

Example 5 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class StreamGraphGeneratorBatchExecutionTest method testCustomManagedMemoryWeights.

@Test
public void testCustomManagedMemoryWeights() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    SingleOutputStreamOperator<Integer> process = env.fromElements(1, 2).keyBy(Integer::intValue).process(DUMMY_PROCESS_FUNCTION);
    DataStreamSink<Integer> sink = process.addSink(new DiscardingSink<>());
    final Configuration configuration = new Configuration();
    configuration.set(ExecutionOptions.SORTED_INPUTS_MEMORY, MemorySize.ofMebiBytes(42));
    StreamGraph graph = getStreamGraphInBatchMode(sink, configuration);
    StreamNode processNode = graph.getStreamNode(process.getId());
    final Map<ManagedMemoryUseCase, Integer> expectedOperatorWeights = new HashMap<>();
    expectedOperatorWeights.put(ManagedMemoryUseCase.OPERATOR, 42);
    assertThat(processNode.getManagedMemoryOperatorScopeUseCaseWeights(), equalTo(expectedOperatorWeights));
    assertThat(processNode.getManagedMemorySlotScopeUseCases(), equalTo(Collections.singleton(ManagedMemoryUseCase.STATE_BACKEND)));
}