Examples with TaskInputsOutputsDescriptor org.apache.flink.runtime.shuffle.TaskInputsOutputsDescriptor used on opensource projects

Example 1 with TaskInputsOutputsDescriptor

use of org.apache.flink.runtime.shuffle.TaskInputsOutputsDescriptor in project flink by apache.

the class SsgNetworkMemoryCalculationUtils method enrichNetworkMemory.

/**
 * Calculates network memory requirement of {@link ExecutionJobVertex} and update {@link
 * ResourceProfile} of corresponding slot sharing group.
 */
public static void enrichNetworkMemory(SlotSharingGroup ssg, Function<JobVertexID, ExecutionJobVertex> ejvs, ShuffleMaster<?> shuffleMaster) {
    ResourceProfile original = ssg.getResourceProfile();
    // supported and the enriching logic only works for 'fine-grained resource management'.
    if (original.equals(ResourceProfile.UNKNOWN) || !original.getNetworkMemory().equals(MemorySize.ZERO)) {
        return;
    }
    MemorySize networkMemory = MemorySize.ZERO;
    for (JobVertexID jvId : ssg.getJobVertexIds()) {
        ExecutionJobVertex ejv = ejvs.apply(jvId);
        TaskInputsOutputsDescriptor desc = buildTaskInputsOutputsDescriptor(ejv, ejvs);
        MemorySize requiredNetworkMemory = shuffleMaster.computeShuffleMemorySizeForTask(desc);
        networkMemory = networkMemory.add(requiredNetworkMemory);
    }
    ResourceProfile enriched = ResourceProfile.newBuilder().setCpuCores(original.getCpuCores()).setTaskHeapMemory(original.getTaskHeapMemory()).setTaskOffHeapMemory(original.getTaskOffHeapMemory()).setManagedMemory(original.getManagedMemory()).setNetworkMemory(networkMemory).setExtendedResources(original.getExtendedResources().values()).build();
    ssg.setResourceProfile(enriched);
}