Examples with MemorySize org.apache.flink.configuration.MemorySize used on opensource projects

Example 21 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class StreamFormatAdapterTest method simpleReadTest.

private void simpleReadTest(int batchSize) throws IOException {
    final Configuration config = new Configuration();
    config.set(StreamFormat.FETCH_IO_SIZE, new MemorySize(batchSize));
    final StreamFormatAdapter<Integer> format = new StreamFormatAdapter<>(new CheckpointedIntFormat());
    final BulkFormat.Reader<Integer> reader = format.createReader(config, new FileSourceSplit("test-id", testPath, 0L, FILE_LEN, 0L, FILE_LEN));
    final List<Integer> result = new ArrayList<>();
    readNumbers(reader, result, NUM_NUMBERS);
    verifyIntListResult(result);
}

Example 22 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class SlotSharingGroupTest method testBuildSlotSharingGroupWithSpecificResource.

@Test
public void testBuildSlotSharingGroupWithSpecificResource() {
    final String name = "ssg";
    final MemorySize heap = MemorySize.ofMebiBytes(100);
    final MemorySize offHeap = MemorySize.ofMebiBytes(200);
    final MemorySize managed = MemorySize.ofMebiBytes(300);
    final SlotSharingGroup slotSharingGroup = SlotSharingGroup.newBuilder(name).setCpuCores(1).setTaskHeapMemory(heap).setTaskOffHeapMemory(offHeap).setManagedMemory(managed).setExternalResource("gpu", 1).build();
    assertThat(slotSharingGroup.getName(), is(name));
    assertThat(slotSharingGroup.getCpuCores().get(), is(1.0));
    assertThat(slotSharingGroup.getTaskHeapMemory().get(), is(heap));
    assertThat(slotSharingGroup.getTaskOffHeapMemory().get(), is(offHeap));
    assertThat(slotSharingGroup.getManagedMemory().get(), is(managed));
    assertThat(slotSharingGroup.getExternalResources(), is(Collections.singletonMap("gpu", 1.0)));
}

Example 23 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class GSRecoverableFsDataOutputStream method createWriteChannel.

private GSChecksumWriteChannel createWriteChannel() {
    // add a new component blob id for the new channel to write to
    UUID componentObjectId = UUID.randomUUID();
    componentObjectIds.add(componentObjectId);
    GSBlobIdentifier blobIdentifier = BlobUtils.getTemporaryBlobIdentifier(finalBlobIdentifier, componentObjectId, options);
    // create the channel, using an explicit chunk size if specified in options
    Optional<MemorySize> writerChunkSize = options.getWriterChunkSize();
    GSBlobStorage.WriteChannel writeChannel = writerChunkSize.isPresent() ? storage.writeBlob(blobIdentifier, writerChunkSize.get()) : storage.writeBlob(blobIdentifier);
    return new GSChecksumWriteChannel(storage, writeChannel, blobIdentifier);
}

Example 24 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class DefaultVertexParallelismDecider method calculateParallelism.

private int calculateParallelism(List<BlockingResultInfo> consumedResults) {
    long broadcastBytes = consumedResults.stream().filter(BlockingResultInfo::isBroadcast).mapToLong(consumedResult -> consumedResult.getBlockingPartitionSizes().stream().reduce(0L, Long::sum)).sum();
    long nonBroadcastBytes = consumedResults.stream().filter(consumedResult -> !consumedResult.isBroadcast()).mapToLong(consumedResult -> consumedResult.getBlockingPartitionSizes().stream().reduce(0L, Long::sum)).sum();
    long expectedMaxBroadcastBytes = (long) Math.ceil((dataVolumePerTask * CAP_RATIO_OF_BROADCAST));
    if (broadcastBytes > expectedMaxBroadcastBytes) {
        LOG.info("The size of broadcast data {} is larger than the expected maximum value {} ('{}' * {})." + " Use {} as the size of broadcast data to decide the parallelism.", new MemorySize(broadcastBytes), new MemorySize(expectedMaxBroadcastBytes), JobManagerOptions.ADAPTIVE_BATCH_SCHEDULER_DATA_VOLUME_PER_TASK.key(), CAP_RATIO_OF_BROADCAST, new MemorySize(expectedMaxBroadcastBytes));
        broadcastBytes = expectedMaxBroadcastBytes;
    }
    int parallelism = (int) Math.ceil((double) nonBroadcastBytes / (dataVolumePerTask - broadcastBytes));
    LOG.debug("The size of broadcast data is {}, the size of non-broadcast data is {}, " + "the initially decided parallelism is {}.", new MemorySize(broadcastBytes), new MemorySize(nonBroadcastBytes), parallelism);
    if (parallelism < minParallelism) {
        LOG.info("The initially decided parallelism {} is smaller than the minimum parallelism {} " + "(which is configured by '{}'). Use {} as the finally decided parallelism.", parallelism, minParallelism, JobManagerOptions.ADAPTIVE_BATCH_SCHEDULER_MIN_PARALLELISM.key(), minParallelism);
        parallelism = minParallelism;
    } else if (parallelism > maxParallelism) {
        LOG.info("The initially decided parallelism {} is larger than the maximum parallelism {} " + "(which is configured by '{}'). Use {} as the finally decided parallelism.", parallelism, maxParallelism, JobManagerOptions.ADAPTIVE_BATCH_SCHEDULER_MAX_PARALLELISM.key(), maxParallelism);
        parallelism = maxParallelism;
    }
    return parallelism;
}

Example 25 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class SsgNetworkMemoryCalculationUtils method enrichNetworkMemory.

/**
 * Calculates network memory requirement of {@link ExecutionJobVertex} and update {@link
 * ResourceProfile} of corresponding slot sharing group.
 */
public static void enrichNetworkMemory(SlotSharingGroup ssg, Function<JobVertexID, ExecutionJobVertex> ejvs, ShuffleMaster<?> shuffleMaster) {
    ResourceProfile original = ssg.getResourceProfile();
    // supported and the enriching logic only works for 'fine-grained resource management'.
    if (original.equals(ResourceProfile.UNKNOWN) || !original.getNetworkMemory().equals(MemorySize.ZERO)) {
        return;
    }
    MemorySize networkMemory = MemorySize.ZERO;
    for (JobVertexID jvId : ssg.getJobVertexIds()) {
        ExecutionJobVertex ejv = ejvs.apply(jvId);
        TaskInputsOutputsDescriptor desc = buildTaskInputsOutputsDescriptor(ejv, ejvs);
        MemorySize requiredNetworkMemory = shuffleMaster.computeShuffleMemorySizeForTask(desc);
        networkMemory = networkMemory.add(requiredNetworkMemory);
    }
    ResourceProfile enriched = ResourceProfile.newBuilder().setCpuCores(original.getCpuCores()).setTaskHeapMemory(original.getTaskHeapMemory()).setTaskOffHeapMemory(original.getTaskOffHeapMemory()).setManagedMemory(original.getManagedMemory()).setNetworkMemory(networkMemory).setExtendedResources(original.getExtendedResources().values()).build();
    ssg.setResourceProfile(enriched);
}