Examples with ManagedMemoryUseCase org.apache.flink.core.memory.ManagedMemoryUseCase used on opensource projects

Example 6 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class ManagedMemoryUtilsTest method testConvertToFractionOfSlot.

@Test
public void testConvertToFractionOfSlot() {
    final ManagedMemoryUseCase useCase = ManagedMemoryUseCase.OPERATOR;
    final double fractionOfUseCase = 0.3;
    final double fractionOfSlot = ManagedMemoryUtils.convertToFractionOfSlot(useCase, fractionOfUseCase, new HashSet<ManagedMemoryUseCase>() {

        {
            add(ManagedMemoryUseCase.OPERATOR);
            add(ManagedMemoryUseCase.STATE_BACKEND);
            add(ManagedMemoryUseCase.PYTHON);
        }
    }, CONFIG_WITH_ALL_USE_CASES, Optional.of(true), ClassLoader.getSystemClassLoader());
    assertEquals(fractionOfUseCase * OPERATOR_WEIGHT / TOTAL_WEIGHT, fractionOfSlot, DELTA);
}

Example 7 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class ManagedMemoryUtilsTest method testConvertToFractionOfSlotWeightNotConfigured.

@Test
public void testConvertToFractionOfSlotWeightNotConfigured() {
    final ManagedMemoryUseCase useCase = ManagedMemoryUseCase.OPERATOR;
    final double fractionOfUseCase = 0.3;
    final Configuration config = new Configuration() {

        {
            set(TaskManagerOptions.MANAGED_MEMORY_CONSUMER_WEIGHTS, Collections.emptyMap());
        }
    };
    final double fractionOfSlot = ManagedMemoryUtils.convertToFractionOfSlot(useCase, fractionOfUseCase, new HashSet<ManagedMemoryUseCase>() {

        {
            add(ManagedMemoryUseCase.OPERATOR);
            add(ManagedMemoryUseCase.STATE_BACKEND);
            add(ManagedMemoryUseCase.PYTHON);
        }
    }, config, Optional.of(true), ClassLoader.getSystemClassLoader());
    assertEquals(0.0, fractionOfSlot, DELTA);
}

Example 8 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class StreamingJobGraphGenerator method setManagedMemoryFractionForSlotSharingGroup.

private static void setManagedMemoryFractionForSlotSharingGroup(final SlotSharingGroup slotSharingGroup, final Map<JobVertexID, Integer> vertexHeadOperators, final Map<JobVertexID, Set<Integer>> vertexOperators, final Map<Integer, StreamConfig> operatorConfigs, final Map<Integer, Map<Integer, StreamConfig>> vertexChainedConfigs, final java.util.function.Function<Integer, Map<ManagedMemoryUseCase, Integer>> operatorScopeManagedMemoryUseCaseWeightsRetriever, final java.util.function.Function<Integer, Set<ManagedMemoryUseCase>> slotScopeManagedMemoryUseCasesRetriever) {
    final Set<Integer> groupOperatorIds = slotSharingGroup.getJobVertexIds().stream().flatMap((vid) -> vertexOperators.get(vid).stream()).collect(Collectors.toSet());
    final Map<ManagedMemoryUseCase, Integer> groupOperatorScopeUseCaseWeights = groupOperatorIds.stream().flatMap((oid) -> operatorScopeManagedMemoryUseCaseWeightsRetriever.apply(oid).entrySet().stream()).collect(Collectors.groupingBy(Map.Entry::getKey, Collectors.summingInt(Map.Entry::getValue)));
    final Set<ManagedMemoryUseCase> groupSlotScopeUseCases = groupOperatorIds.stream().flatMap((oid) -> slotScopeManagedMemoryUseCasesRetriever.apply(oid).stream()).collect(Collectors.toSet());
    for (JobVertexID jobVertexID : slotSharingGroup.getJobVertexIds()) {
        for (int operatorNodeId : vertexOperators.get(jobVertexID)) {
            final StreamConfig operatorConfig = operatorConfigs.get(operatorNodeId);
            final Map<ManagedMemoryUseCase, Integer> operatorScopeUseCaseWeights = operatorScopeManagedMemoryUseCaseWeightsRetriever.apply(operatorNodeId);
            final Set<ManagedMemoryUseCase> slotScopeUseCases = slotScopeManagedMemoryUseCasesRetriever.apply(operatorNodeId);
            setManagedMemoryFractionForOperator(operatorScopeUseCaseWeights, slotScopeUseCases, groupOperatorScopeUseCaseWeights, groupSlotScopeUseCases, operatorConfig);
        }
        // need to refresh the chained task configs because they are serialized
        final int headOperatorNodeId = vertexHeadOperators.get(jobVertexID);
        final StreamConfig vertexConfig = operatorConfigs.get(headOperatorNodeId);
        vertexConfig.setTransitiveChainedTaskConfigs(vertexChainedConfigs.get(headOperatorNodeId));
    }
}

Example 9 with ManagedMemoryUseCase

use of org.apache.flink.core.memory.ManagedMemoryUseCase in project flink by apache.

the class StreamingJobGraphGenerator method setManagedMemoryFractionForOperator.

private static void setManagedMemoryFractionForOperator(final Map<ManagedMemoryUseCase, Integer> operatorScopeUseCaseWeights, final Set<ManagedMemoryUseCase> slotScopeUseCases, final Map<ManagedMemoryUseCase, Integer> groupManagedMemoryWeights, final Set<ManagedMemoryUseCase> groupSlotScopeUseCases, final StreamConfig operatorConfig) {
    // learn which use cases exist in the group from either one of the stream configs.
    for (Map.Entry<ManagedMemoryUseCase, Integer> entry : groupManagedMemoryWeights.entrySet()) {
        final ManagedMemoryUseCase useCase = entry.getKey();
        final int groupWeight = entry.getValue();
        final int operatorWeight = operatorScopeUseCaseWeights.getOrDefault(useCase, 0);
        operatorConfig.setManagedMemoryFractionOperatorOfUseCase(useCase, operatorWeight > 0 ? ManagedMemoryUtils.getFractionRoundedDown(operatorWeight, groupWeight) : 0.0);
    }
    for (ManagedMemoryUseCase useCase : groupSlotScopeUseCases) {
        operatorConfig.setManagedMemoryFractionOperatorOfUseCase(useCase, slotScopeUseCases.contains(useCase) ? 1.0 : 0.0);
    }
}