Examples with MemorySize org.apache.flink.configuration.MemorySize used on opensource projects

Example 46 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class BufferDebloatConfigurationTest method testMinGreaterThanMaxBufferSize.

@Test
public void testMinGreaterThanMaxBufferSize() {
    final Configuration config = new Configuration();
    config.set(TaskManagerOptions.MIN_MEMORY_SEGMENT_SIZE, new MemorySize(50));
    config.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, new MemorySize(49));
    assertThrows(IllegalArgumentException.class, () -> BufferDebloatConfiguration.fromConfiguration(config));
}

Example 47 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class BucketAssignerITCases method testAssembleBucketPath.

@Test
public void testAssembleBucketPath() throws Exception {
    final File outDir = TEMP_FOLDER.newFolder();
    final Path basePath = new Path(outDir.toURI());
    final long time = 1000L;
    final RollingPolicy<String, String> rollingPolicy = DefaultRollingPolicy.builder().withMaxPartSize(new MemorySize(7L)).build();
    final Buckets<String, String> buckets = new Buckets<>(basePath, new BasePathBucketAssigner<>(), new DefaultBucketFactoryImpl<>(), new RowWiseBucketWriter<>(FileSystem.get(basePath.toUri()).createRecoverableWriter(), new SimpleStringEncoder<>()), rollingPolicy, 0, OutputFileConfig.builder().build());
    Bucket<String, String> bucket = buckets.onElement("abc", new TestUtils.MockSinkContext(time, time, time));
    Assert.assertEquals(new Path(basePath.toUri()), bucket.getBucketPath());
}

Example 48 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class BucketsTest method testMergeAtScaleInAndMaxCounterAtRecovery.

@Test
public void testMergeAtScaleInAndMaxCounterAtRecovery() throws Exception {
    final File outDir = TEMP_FOLDER.newFolder();
    final Path path = new Path(outDir.toURI());
    final RollingPolicy<String, String> onCheckpointRP = DefaultRollingPolicy.builder().withMaxPartSize(// roll with 2 elements
    new MemorySize(7L)).build();
    final MockListState<byte[]> bucketStateContainerOne = new MockListState<>();
    final MockListState<byte[]> bucketStateContainerTwo = new MockListState<>();
    final MockListState<Long> partCounterContainerOne = new MockListState<>();
    final MockListState<Long> partCounterContainerTwo = new MockListState<>();
    final Buckets<String, String> bucketsOne = createBuckets(path, onCheckpointRP, 0);
    final Buckets<String, String> bucketsTwo = createBuckets(path, onCheckpointRP, 1);
    bucketsOne.onElement("test1", new TestUtils.MockSinkContext(null, 1L, 2L));
    bucketsOne.snapshotState(0L, bucketStateContainerOne, partCounterContainerOne);
    Assert.assertEquals(1L, bucketsOne.getMaxPartCounter());
    // make sure we have one in-progress file here
    Assert.assertNotNull(bucketsOne.getActiveBuckets().get("test1").getInProgressPart());
    // add a couple of in-progress files so that the part counter increases.
    bucketsTwo.onElement("test1", new TestUtils.MockSinkContext(null, 1L, 2L));
    bucketsTwo.onElement("test1", new TestUtils.MockSinkContext(null, 1L, 2L));
    bucketsTwo.onElement("test1", new TestUtils.MockSinkContext(null, 1L, 2L));
    bucketsTwo.snapshotState(0L, bucketStateContainerTwo, partCounterContainerTwo);
    Assert.assertEquals(2L, bucketsTwo.getMaxPartCounter());
    // make sure we have one in-progress file here and a pending
    Assert.assertEquals(1L, bucketsTwo.getActiveBuckets().get("test1").getPendingFileRecoverablesPerCheckpoint().size());
    Assert.assertNotNull(bucketsTwo.getActiveBuckets().get("test1").getInProgressPart());
    final ListState<byte[]> mergedBucketStateContainer = new MockListState<>();
    final ListState<Long> mergedPartCounterContainer = new MockListState<>();
    mergedBucketStateContainer.addAll(bucketStateContainerOne.getBackingList());
    mergedBucketStateContainer.addAll(bucketStateContainerTwo.getBackingList());
    mergedPartCounterContainer.addAll(partCounterContainerOne.getBackingList());
    mergedPartCounterContainer.addAll(partCounterContainerTwo.getBackingList());
    final Buckets<String, String> restoredBuckets = restoreBuckets(path, onCheckpointRP, 0, mergedBucketStateContainer, mergedPartCounterContainer);
    // we get the maximum of the previous tasks
    Assert.assertEquals(2L, restoredBuckets.getMaxPartCounter());
    final Map<String, Bucket<String, String>> activeBuckets = restoredBuckets.getActiveBuckets();
    Assert.assertEquals(1L, activeBuckets.size());
    Assert.assertTrue(activeBuckets.keySet().contains("test1"));
    final Bucket<String, String> bucket = activeBuckets.get("test1");
    Assert.assertEquals("test1", bucket.getBucketId());
    Assert.assertEquals(new Path(path, "test1"), bucket.getBucketPath());
    // the restored part file
    Assert.assertNotNull(bucket.getInProgressPart());
    // this is due to the Bucket#merge(). The in progress file of one
    // of the previous tasks is put in the list of pending files.
    Assert.assertEquals(1L, bucket.getPendingFileRecoverablesForCurrentCheckpoint().size());
    // we commit the pending for previous checkpoints
    Assert.assertTrue(bucket.getPendingFileRecoverablesPerCheckpoint().isEmpty());
}

Example 49 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class JobManagerFlinkMemoryUtils method deriveFromRequiredFineGrainedOptions.

@Override
public JobManagerFlinkMemory deriveFromRequiredFineGrainedOptions(Configuration config) {
    MemorySize jvmHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.JVM_HEAP_MEMORY);
    MemorySize offHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.OFF_HEAP_MEMORY);
    if (config.contains(JobManagerOptions.TOTAL_FLINK_MEMORY)) {
        // derive network memory from total flink memory, and check against network min/max
        MemorySize totalFlinkMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.TOTAL_FLINK_MEMORY);
        if (config.contains(JobManagerOptions.OFF_HEAP_MEMORY)) {
            // off-heap memory is explicitly set by user
            sanityCheckTotalFlinkMemory(totalFlinkMemorySize, jvmHeapMemorySize, offHeapMemorySize);
        } else {
            // off-heap memory is not explicitly set by user, derive it from Total Flink Memory
            // and JVM Heap
            offHeapMemorySize = deriveOffHeapMemory(jvmHeapMemorySize, totalFlinkMemorySize, offHeapMemorySize);
        }
    }
    return createJobManagerFlinkMemory(jvmHeapMemorySize, offHeapMemorySize);
}

Example 50 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class TaskExecutorFlinkMemoryUtils method deriveFromTotalFlinkMemory.

@Override
public TaskExecutorFlinkMemory deriveFromTotalFlinkMemory(final Configuration config, final MemorySize totalFlinkMemorySize) {
    final MemorySize frameworkHeapMemorySize = getFrameworkHeapMemorySize(config);
    final MemorySize frameworkOffHeapMemorySize = getFrameworkOffHeapMemorySize(config);
    final MemorySize taskOffHeapMemorySize = getTaskOffHeapMemorySize(config);
    final MemorySize taskHeapMemorySize;
    final MemorySize networkMemorySize;
    final MemorySize managedMemorySize;
    if (isTaskHeapMemorySizeExplicitlyConfigured(config)) {
        // task heap memory is configured,
        // derive managed memory first, leave the remaining to network memory and check against
        // network min/max
        taskHeapMemorySize = getTaskHeapMemorySize(config);
        managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize);
        final MemorySize totalFlinkExcludeNetworkMemorySize = frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize);
        if (totalFlinkExcludeNetworkMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
            throw new IllegalConfigurationException("Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString() + "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString() + "), Task Heap Memory (" + taskHeapMemorySize.toHumanReadableString() + "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString() + ") and Managed Memory (" + managedMemorySize.toHumanReadableString() + ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
        }
        networkMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeNetworkMemorySize);
        sanityCheckNetworkMemoryWithExplicitlySetTotalFlinkAndHeapMemory(config, networkMemorySize, totalFlinkMemorySize);
    } else {
        // task heap memory is not configured
        // derive managed memory and network memory, leave the remaining to task heap memory
        managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize);
        networkMemorySize = isUsingLegacyNetworkConfigs(config) ? getNetworkMemorySizeWithLegacyConfig(config) : deriveNetworkMemoryWithFraction(config, totalFlinkMemorySize);
        final MemorySize totalFlinkExcludeTaskHeapMemorySize = frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize).add(networkMemorySize);
        if (totalFlinkExcludeTaskHeapMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
            throw new IllegalConfigurationException("Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString() + "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString() + "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString() + "), Managed Memory (" + managedMemorySize.toHumanReadableString() + ") and Network Memory (" + networkMemorySize.toHumanReadableString() + ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
        }
        taskHeapMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeTaskHeapMemorySize);
    }
    final TaskExecutorFlinkMemory flinkInternalMemory = new TaskExecutorFlinkMemory(frameworkHeapMemorySize, frameworkOffHeapMemorySize, taskHeapMemorySize, taskOffHeapMemorySize, networkMemorySize, managedMemorySize);
    sanityCheckTotalFlinkMemory(config, flinkInternalMemory);
    return flinkInternalMemory;
}