Examples with MemorySize org.apache.flink.configuration.MemorySize used on opensource projects

Example 1 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class WebFrontendITCase method getConfiguration.

@Test
public void getConfiguration() throws Exception {
    String config = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/jobmanager/config");
    Map<String, String> conf = WebMonitorUtils.fromKeyValueJsonArray(config);
    MemorySize expected = CLUSTER_CONFIGURATION.get(TaskManagerOptions.MANAGED_MEMORY_SIZE);
    MemorySize actual = MemorySize.parse(conf.get(TaskManagerOptions.MANAGED_MEMORY_SIZE.key()));
    assertEquals(expected, actual);
}

Example 2 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class BootstrapToolsTest method testGetTaskManagerShellCommand.

@Test
public void testGetTaskManagerShellCommand() {
    final Configuration cfg = new Configuration();
    final TaskExecutorProcessSpec taskExecutorProcessSpec = new TaskExecutorProcessSpec(new CPUResource(1.0), // frameworkHeapSize
    new MemorySize(0), // frameworkOffHeapSize
    new MemorySize(0), // taskHeapSize
    new MemorySize(111), // taskOffHeapSize
    new MemorySize(0), // networkMemSize
    new MemorySize(222), // managedMemorySize
    new MemorySize(0), // jvmMetaspaceSize
    new MemorySize(333), // jvmOverheadSize
    new MemorySize(0), Collections.emptyList());
    final ContaineredTaskManagerParameters containeredParams = new ContaineredTaskManagerParameters(taskExecutorProcessSpec, new HashMap<String, String>());
    // no logging, with/out krb5
    final String java = "$JAVA_HOME/bin/java";
    final String jvmmem = "-Xmx111 -Xms111 -XX:MaxDirectMemorySize=222 -XX:MaxMetaspaceSize=333";
    // if set
    final String jvmOpts = "-Djvm";
    // if set
    final String tmJvmOpts = "-DtmJvm";
    // if set
    final String logfile = "-Dlog.file=./logs/taskmanager.log";
    // if set
    final String logback = "-Dlogback.configurationFile=file:./conf/logback.xml";
    final String log4j = "-Dlog4j.configuration=file:./conf/log4j.properties" + // if set
    " -Dlog4j.configurationFile=file:./conf/log4j.properties";
    final String mainClass = "org.apache.flink.runtime.clusterframework.BootstrapToolsTest";
    final String dynamicConfigs = TaskExecutorProcessUtils.generateDynamicConfigsStr(taskExecutorProcessSpec).trim();
    final String basicArgs = "--configDir ./conf";
    final String mainArgs = "-Djobmanager.rpc.address=host1 -Dkey.a=v1";
    final String args = dynamicConfigs + " " + basicArgs + " " + mainArgs;
    final String redirects = "1> ./logs/taskmanager.out 2> ./logs/taskmanager.err";
    assertEquals(java + " " + jvmmem + "" + // jvmOpts
    "" + // logging
    " " + mainClass + " " + dynamicConfigs + " " + basicArgs + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, false, false, this.getClass(), ""));
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + // logging
    "" + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, false, false, this.getClass(), mainArgs));
    final String krb5 = "-Djava.security.krb5.conf=krb5.conf";
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + // logging
    "" + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, false, true, this.getClass(), mainArgs));
    // logback only, with/out krb5
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + " " + logfile + " " + logback + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, false, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + " " + logfile + " " + logback + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, false, true, this.getClass(), mainArgs));
    // log4j, with/out krb5
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + " " + logfile + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + " " + logfile + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, true, true, this.getClass(), mainArgs));
    // logback + log4j, with/out krb5
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    // logback + log4j, with/out krb5, different JVM opts
    cfg.setString(CoreOptions.FLINK_JVM_OPTIONS, jvmOpts);
    assertEquals(java + " " + jvmmem + " " + jvmOpts + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + jvmOpts + " " + krb5 + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    // logback + log4j, with/out krb5, different JVM opts
    cfg.setString(CoreOptions.FLINK_TM_JVM_OPTIONS, tmJvmOpts);
    assertEquals(java + " " + jvmmem + " " + jvmOpts + " " + tmJvmOpts + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + jvmOpts + " " + tmJvmOpts + " " + krb5 + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    // now try some configurations with different yarn.container-start-command-template
    cfg.setString(ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, "%java% 1 %jvmmem% 2 %jvmopts% 3 %logging% 4 %class% 5 %args% 6 %redirects%");
    assertEquals(// jvmOpts
    java + " 1 " + jvmmem + " 2 " + jvmOpts + " " + tmJvmOpts + " " + krb5 + " 3 " + logfile + " " + logback + " " + log4j + " 4 " + mainClass + " 5 " + args + " 6 " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    cfg.setString(ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, "%java% %logging% %jvmopts% %jvmmem% %class% %args% %redirects%");
    assertEquals(java + " " + logfile + " " + logback + " " + log4j + " " + jvmOpts + " " + tmJvmOpts + " " + // jvmOpts
    krb5 + " " + jvmmem + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
}

Example 3 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class TaskExecutorProcessUtilsTest method testConfigNetworkMemoryFraction.

@Test
public void testConfigNetworkMemoryFraction() {
    final MemorySize networkMin = MemorySize.ZERO;
    final MemorySize networkMax = MemorySize.parse("1t");
    final float fraction = 0.2f;
    Configuration conf = new Configuration();
    conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
    conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);
    conf.setFloat(TaskManagerOptions.NETWORK_MEMORY_FRACTION, fraction);
    // validate in configurations without explicit total flink/process memory, otherwise
    // explicit configured
    // network memory fraction might conflict with total flink/process memory minus other memory
    // sizes
    validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(taskExecutorProcessSpec.getTotalFlinkMemorySize().multiply(fraction))));
}

Example 4 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class TaskExecutorProcessUtilsTest method testConfigTaskHeapMemory.

@Test
public void testConfigTaskHeapMemory() {
    final MemorySize taskHeapSize = MemorySize.parse("50m");
    Configuration conf = new Configuration();
    conf.set(TaskManagerOptions.TASK_HEAP_MEMORY, taskHeapSize);
    // validate in configurations without explicit task heap memory size,
    // to avoid checking against overwritten task heap memory size
    validateInConfigurationsWithoutExplicitTaskHeapMem(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getTaskHeapSize(), is(taskHeapSize)));
}

Example 5 with MemorySize

use of org.apache.flink.configuration.MemorySize in project flink by apache.

the class ResourceProfileTest method testMergeWithOverflow.

@Test
public void testMergeWithOverflow() {
    final CPUResource largeDouble = new CPUResource(Double.MAX_VALUE - 1.0);
    final MemorySize largeMemory = MemorySize.MAX_VALUE.subtract(MemorySize.parse("100m"));
    final ResourceProfile rp1 = ResourceProfile.newBuilder().setCpuCores(3.0).setTaskHeapMemoryMB(300).setTaskOffHeapMemoryMB(300).setManagedMemoryMB(300).setNetworkMemoryMB(300).build();
    final ResourceProfile rp2 = ResourceProfile.newBuilder().setCpuCores(largeDouble).setTaskHeapMemory(largeMemory).setTaskOffHeapMemory(largeMemory).setManagedMemory(largeMemory).setNetworkMemory(largeMemory).build();
    List<ArithmeticException> exceptions = new ArrayList<>();
    try {
        rp2.merge(rp2);
    } catch (ArithmeticException e) {
        exceptions.add(e);
    }
    try {
        rp2.merge(rp1);
    } catch (ArithmeticException e) {
        exceptions.add(e);
    }
    try {
        rp1.merge(rp2);
    } catch (ArithmeticException e) {
        exceptions.add(e);
    }
    assertEquals(3, exceptions.size());
}