Examples with MemorySize org.apache.flink.configuration.MemorySize used on opensource projects

Example 76 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class TaskExecutorResourceUtilsTest method testCalculateTotalFlinkMemoryWithMissingFactors.

@Test(expected = IllegalArgumentException.class)
public void testCalculateTotalFlinkMemoryWithMissingFactors() {
    Configuration config = new Configuration();
    config.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, new MemorySize(1));
    config.set(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY, new MemorySize(3));
    config.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, new MemorySize(4));
    config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, new MemorySize(7));
    TaskExecutorResourceUtils.calculateTotalFlinkMemoryFromComponents(config);
}

Example 77 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class ProcessMemoryUtils method deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory.

public JvmMetaspaceAndOverhead deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory(Configuration config, MemorySize totalFlinkMemorySize) {
    MemorySize jvmMetaspaceSize = getMemorySizeFromConfig(config, options.getJvmOptions().getJvmMetaspaceOption());
    MemorySize totalFlinkAndJvmMetaspaceSize = totalFlinkMemorySize.add(jvmMetaspaceSize);
    JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead;
    if (config.contains(options.getTotalProcessMemoryOption())) {
        MemorySize jvmOverheadSize = deriveJvmOverheadFromTotalFlinkMemoryAndOtherComponents(config, totalFlinkMemorySize);
        jvmMetaspaceAndOverhead = new JvmMetaspaceAndOverhead(jvmMetaspaceSize, jvmOverheadSize);
    } else {
        MemorySize jvmOverheadSize = deriveWithInverseFraction("jvm overhead memory", totalFlinkAndJvmMetaspaceSize, getJvmOverheadRangeFraction(config));
        jvmMetaspaceAndOverhead = new JvmMetaspaceAndOverhead(jvmMetaspaceSize, jvmOverheadSize);
        sanityCheckTotalProcessMemory(config, totalFlinkMemorySize, jvmMetaspaceAndOverhead);
    }
    return jvmMetaspaceAndOverhead;
}

Example 78 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class ProcessMemoryUtils method deriveJvmOverheadFromTotalFlinkMemoryAndOtherComponents.

private MemorySize deriveJvmOverheadFromTotalFlinkMemoryAndOtherComponents(Configuration config, MemorySize totalFlinkMemorySize) {
    MemorySize totalProcessMemorySize = getMemorySizeFromConfig(config, options.getTotalProcessMemoryOption());
    MemorySize jvmMetaspaceSize = getMemorySizeFromConfig(config, options.getJvmOptions().getJvmMetaspaceOption());
    MemorySize totalFlinkAndJvmMetaspaceSize = totalFlinkMemorySize.add(jvmMetaspaceSize);
    if (totalProcessMemorySize.getBytes() < totalFlinkAndJvmMetaspaceSize.getBytes()) {
        throw new IllegalConfigurationException("The configured Total Process Memory size (%s) is less than the sum of the derived " + "Total Flink Memory size (%s) and the configured or default JVM Metaspace size  (%s).", totalProcessMemorySize.toHumanReadableString(), totalFlinkMemorySize.toHumanReadableString(), jvmMetaspaceSize.toHumanReadableString());
    }
    MemorySize jvmOverheadSize = totalProcessMemorySize.subtract(totalFlinkAndJvmMetaspaceSize);
    sanityCheckJvmOverhead(config, jvmOverheadSize, totalProcessMemorySize);
    return jvmOverheadSize;
}

Example 79 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class CheckpointStorageLoaderTest method testLoadFileSystemCheckpointStorage.

// ------------------------------------------------------------------------
// File System Checkpoint Storage
// ------------------------------------------------------------------------
/**
 * Validates loading a file system checkpoint storage with additional parameters from the
 * cluster configuration.
 */
@Test
public void testLoadFileSystemCheckpointStorage() throws Exception {
    final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
    final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
    final Path expectedCheckpointsPath = new Path(checkpointDir);
    final Path expectedSavepointsPath = new Path(savepointDir);
    final MemorySize threshold = MemorySize.parse("900kb");
    final int minWriteBufferSize = 1024;
    // we configure with the explicit string (rather than
    // AbstractStateBackend#X_STATE_BACKEND_NAME)
    // to guard against config-breaking changes of the name
    final Configuration config1 = new Configuration();
    config1.set(CheckpointingOptions.CHECKPOINT_STORAGE, "filesystem");
    config1.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
    config1.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
    config1.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
    config1.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, minWriteBufferSize);
    CheckpointStorage storage1 = CheckpointStorageLoader.fromConfig(config1, cl, null).get();
    Assert.assertThat(storage1, Matchers.instanceOf(FileSystemCheckpointStorage.class));
    FileSystemCheckpointStorage fs1 = (FileSystemCheckpointStorage) storage1;
    Assert.assertThat(fs1.getCheckpointPath(), normalizedPath(expectedCheckpointsPath));
    Assert.assertThat(fs1.getSavepointPath(), normalizedPath(expectedSavepointsPath));
    Assert.assertEquals(threshold.getBytes(), fs1.getMinFileSizeThreshold());
    Assert.assertEquals(Math.max(threshold.getBytes(), minWriteBufferSize), fs1.getWriteBufferSize());
}

Example 80 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class StateBackendLoadingTest method testLoadFileSystemStateBackend.

// ------------------------------------------------------------------------
// File System State Backend
// ------------------------------------------------------------------------
/**
 * Validates loading a file system state backend with additional parameters from the cluster
 * configuration.
 */
@Test
public void testLoadFileSystemStateBackend() throws Exception {
    final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
    final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
    final Path expectedCheckpointsPath = new Path(checkpointDir);
    final Path expectedSavepointsPath = new Path(savepointDir);
    final MemorySize threshold = MemorySize.parse("900kb");
    final int minWriteBufferSize = 1024;
    // we configure with the explicit string (rather than
    // AbstractStateBackend#X_STATE_BACKEND_NAME)
    // to guard against config-breaking changes of the name
    final Configuration config1 = new Configuration();
    config1.setString(backendKey, "filesystem");
    config1.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
    config1.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
    config1.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
    config1.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, minWriteBufferSize);
    final Configuration config2 = new Configuration();
    config2.setString(backendKey, FsStateBackendFactory.class.getName());
    config2.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
    config2.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
    config2.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
    config1.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, minWriteBufferSize);
    StateBackend backend1 = StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
    StateBackend backend2 = StateBackendLoader.loadStateBackendFromConfig(config2, cl, null);
    assertTrue(backend1 instanceof HashMapStateBackend);
    assertTrue(backend2 instanceof FsStateBackend);
    HashMapStateBackend fs1 = (HashMapStateBackend) backend1;
    FsStateBackend fs2 = (FsStateBackend) backend2;
    assertEquals(expectedCheckpointsPath, fs2.getCheckpointPath());
    assertEquals(expectedSavepointsPath, fs2.getSavepointPath());
    assertEquals(threshold.getBytes(), fs2.getMinFileSizeThreshold());
    assertEquals(Math.max(threshold.getBytes(), minWriteBufferSize), fs2.getWriteBufferSize());
}