Examples with CPUResource org.apache.flink.api.common.resources.CPUResource used on opensource projects

Example 1 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class BootstrapToolsTest method testGetTaskManagerShellCommand.

@Test
public void testGetTaskManagerShellCommand() {
    final Configuration cfg = new Configuration();
    final TaskExecutorProcessSpec taskExecutorProcessSpec = new TaskExecutorProcessSpec(new CPUResource(1.0), // frameworkHeapSize
    new MemorySize(0), // frameworkOffHeapSize
    new MemorySize(0), // taskHeapSize
    new MemorySize(111), // taskOffHeapSize
    new MemorySize(0), // networkMemSize
    new MemorySize(222), // managedMemorySize
    new MemorySize(0), // jvmMetaspaceSize
    new MemorySize(333), // jvmOverheadSize
    new MemorySize(0), Collections.emptyList());
    final ContaineredTaskManagerParameters containeredParams = new ContaineredTaskManagerParameters(taskExecutorProcessSpec, new HashMap<String, String>());
    // no logging, with/out krb5
    final String java = "$JAVA_HOME/bin/java";
    final String jvmmem = "-Xmx111 -Xms111 -XX:MaxDirectMemorySize=222 -XX:MaxMetaspaceSize=333";
    // if set
    final String jvmOpts = "-Djvm";
    // if set
    final String tmJvmOpts = "-DtmJvm";
    // if set
    final String logfile = "-Dlog.file=./logs/taskmanager.log";
    // if set
    final String logback = "-Dlogback.configurationFile=file:./conf/logback.xml";
    final String log4j = "-Dlog4j.configuration=file:./conf/log4j.properties" + // if set
    " -Dlog4j.configurationFile=file:./conf/log4j.properties";
    final String mainClass = "org.apache.flink.runtime.clusterframework.BootstrapToolsTest";
    final String dynamicConfigs = TaskExecutorProcessUtils.generateDynamicConfigsStr(taskExecutorProcessSpec).trim();
    final String basicArgs = "--configDir ./conf";
    final String mainArgs = "-Djobmanager.rpc.address=host1 -Dkey.a=v1";
    final String args = dynamicConfigs + " " + basicArgs + " " + mainArgs;
    final String redirects = "1> ./logs/taskmanager.out 2> ./logs/taskmanager.err";
    assertEquals(java + " " + jvmmem + "" + // jvmOpts
    "" + // logging
    " " + mainClass + " " + dynamicConfigs + " " + basicArgs + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, false, false, this.getClass(), ""));
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + // logging
    "" + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, false, false, this.getClass(), mainArgs));
    final String krb5 = "-Djava.security.krb5.conf=krb5.conf";
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + // logging
    "" + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, false, true, this.getClass(), mainArgs));
    // logback only, with/out krb5
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + " " + logfile + " " + logback + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, false, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + " " + logfile + " " + logback + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, false, true, this.getClass(), mainArgs));
    // log4j, with/out krb5
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + " " + logfile + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + " " + logfile + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", false, true, true, this.getClass(), mainArgs));
    // logback + log4j, with/out krb5
    assertEquals(// jvmOpts
    java + " " + jvmmem + "" + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + krb5 + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    // logback + log4j, with/out krb5, different JVM opts
    cfg.setString(CoreOptions.FLINK_JVM_OPTIONS, jvmOpts);
    assertEquals(java + " " + jvmmem + " " + jvmOpts + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + jvmOpts + " " + krb5 + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    // logback + log4j, with/out krb5, different JVM opts
    cfg.setString(CoreOptions.FLINK_TM_JVM_OPTIONS, tmJvmOpts);
    assertEquals(java + " " + jvmmem + " " + jvmOpts + " " + tmJvmOpts + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, false, this.getClass(), mainArgs));
    assertEquals(// jvmOpts
    java + " " + jvmmem + " " + jvmOpts + " " + tmJvmOpts + " " + krb5 + " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    // now try some configurations with different yarn.container-start-command-template
    cfg.setString(ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, "%java% 1 %jvmmem% 2 %jvmopts% 3 %logging% 4 %class% 5 %args% 6 %redirects%");
    assertEquals(// jvmOpts
    java + " 1 " + jvmmem + " 2 " + jvmOpts + " " + tmJvmOpts + " " + krb5 + " 3 " + logfile + " " + logback + " " + log4j + " 4 " + mainClass + " 5 " + args + " 6 " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
    cfg.setString(ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, "%java% %logging% %jvmopts% %jvmmem% %class% %args% %redirects%");
    assertEquals(java + " " + logfile + " " + logback + " " + log4j + " " + jvmOpts + " " + tmJvmOpts + " " + // jvmOpts
    krb5 + " " + jvmmem + " " + mainClass + " " + args + " " + redirects, BootstrapTools.getTaskManagerShellCommand(cfg, containeredParams, "./conf", "./logs", true, true, true, this.getClass(), mainArgs));
}

Example 2 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class ResourceProfileTest method testGet.

@Test
public void testGet() {
    ResourceSpec rs = ResourceSpec.newBuilder(1.0, 100).setExtendedResource(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1.6)).build();
    ResourceProfile rp = ResourceProfile.fromResourceSpec(rs, MemorySize.ofMebiBytes(50));
    assertEquals(new CPUResource(1.0), rp.getCpuCores());
    assertEquals(150, rp.getTotalMemory().getMebiBytes());
    assertEquals(100, rp.getOperatorsMemory().getMebiBytes());
    assertEquals(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1.6), rp.getExtendedResources().get(EXTERNAL_RESOURCE_NAME));
}

Example 3 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class ResourceProfileTest method testMergeWithOverflow.

@Test
public void testMergeWithOverflow() {
    final CPUResource largeDouble = new CPUResource(Double.MAX_VALUE - 1.0);
    final MemorySize largeMemory = MemorySize.MAX_VALUE.subtract(MemorySize.parse("100m"));
    final ResourceProfile rp1 = ResourceProfile.newBuilder().setCpuCores(3.0).setTaskHeapMemoryMB(300).setTaskOffHeapMemoryMB(300).setManagedMemoryMB(300).setNetworkMemoryMB(300).build();
    final ResourceProfile rp2 = ResourceProfile.newBuilder().setCpuCores(largeDouble).setTaskHeapMemory(largeMemory).setTaskOffHeapMemory(largeMemory).setManagedMemory(largeMemory).setNetworkMemory(largeMemory).build();
    List<ArithmeticException> exceptions = new ArrayList<>();
    try {
        rp2.merge(rp2);
    } catch (ArithmeticException e) {
        exceptions.add(e);
    }
    try {
        rp2.merge(rp1);
    } catch (ArithmeticException e) {
        exceptions.add(e);
    }
    try {
        rp1.merge(rp2);
    } catch (ArithmeticException e) {
        exceptions.add(e);
    }
    assertEquals(3, exceptions.size());
}

Example 4 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class TaskExecutorProcessUtilsTest method testConfigNoCpuCores.

@Test
public void testConfigNoCpuCores() {
    Configuration conf = new Configuration();
    conf.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 3);
    validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getCpuCores(), is(new CPUResource(3.0))));
}

Example 5 with CPUResource

use of org.apache.flink.api.common.resources.CPUResource in project flink by apache.

the class KubernetesWorkerResourceSpecFactoryTest method testGetCpuCoresNumSlots.

@Test
public void testGetCpuCoresNumSlots() {
    final Configuration configuration = new Configuration();
    configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 3);
    assertThat(KubernetesWorkerResourceSpecFactory.getDefaultCpus(configuration), is(new CPUResource(3.0)));
}