Examples with WorkerResourceSpec org.apache.flink.runtime.resourcemanager.WorkerResourceSpec used on opensource projects

Example 6 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class PendingWorkerCounterTest method testPendingWorkerCounterDecreaseOnZero.

@Test(expected = IllegalStateException.class)
public void testPendingWorkerCounterDecreaseOnZero() {
    final WorkerResourceSpec spec = new WorkerResourceSpec.Builder().build();
    final PendingWorkerCounter counter = new PendingWorkerCounter();
    counter.decreaseAndGet(spec);
}

Example 7 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class PendingWorkerCounterTest method testPendingWorkerCounterIncreaseAndDecrease.

@Test
public void testPendingWorkerCounterIncreaseAndDecrease() {
    final WorkerResourceSpec spec1 = new WorkerResourceSpec.Builder().setCpuCores(1.0).build();
    final WorkerResourceSpec spec2 = new WorkerResourceSpec.Builder().setCpuCores(2.0).build();
    final PendingWorkerCounter counter = new PendingWorkerCounter();
    assertThat(counter.getTotalNum(), is(0));
    assertThat(counter.getNum(spec1), is(0));
    assertThat(counter.getNum(spec2), is(0));
    assertThat(counter.increaseAndGet(spec1), is(1));
    assertThat(counter.getTotalNum(), is(1));
    assertThat(counter.getNum(spec1), is(1));
    assertThat(counter.getNum(spec2), is(0));
    assertThat(counter.increaseAndGet(spec1), is(2));
    assertThat(counter.getTotalNum(), is(2));
    assertThat(counter.getNum(spec1), is(2));
    assertThat(counter.getNum(spec2), is(0));
    assertThat(counter.increaseAndGet(spec2), is(1));
    assertThat(counter.getTotalNum(), is(3));
    assertThat(counter.getNum(spec1), is(2));
    assertThat(counter.getNum(spec2), is(1));
    assertThat(counter.decreaseAndGet(spec1), is(1));
    assertThat(counter.getTotalNum(), is(2));
    assertThat(counter.getNum(spec1), is(1));
    assertThat(counter.getNum(spec2), is(1));
    assertThat(counter.decreaseAndGet(spec2), is(0));
    assertThat(counter.getTotalNum(), is(1));
    assertThat(counter.getNum(spec1), is(1));
    assertThat(counter.getNum(spec2), is(0));
}

Example 8 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class ActiveResourceManager method requestWorkerIfRequired.

private void requestWorkerIfRequired() {
    for (Map.Entry<WorkerResourceSpec, Integer> entry : getRequiredResources().entrySet()) {
        final WorkerResourceSpec workerResourceSpec = entry.getKey();
        final int requiredCount = entry.getValue();
        while (requiredCount > pendingWorkerCounter.getNum(workerResourceSpec)) {
            requestNewWorker(workerResourceSpec);
        }
    }
}

Example 9 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class ActiveResourceManager method onWorkerRegistered.

@Override
protected void onWorkerRegistered(WorkerType worker) {
    final ResourceID resourceId = worker.getResourceID();
    log.info("Worker {} is registered.", resourceId.getStringWithMetadata());
    final WorkerResourceSpec workerResourceSpec = currentAttemptUnregisteredWorkers.remove(resourceId);
    previousAttemptUnregisteredWorkers.remove(resourceId);
    if (workerResourceSpec != null) {
        final int count = pendingWorkerCounter.decreaseAndGet(workerResourceSpec);
        log.info("Worker {} with resource spec {} was requested in current attempt." + " Current pending count after registering: {}.", resourceId.getStringWithMetadata(), workerResourceSpec, count);
    }
}

Example 10 with WorkerResourceSpec

use of org.apache.flink.runtime.resourcemanager.WorkerResourceSpec in project flink by apache.

the class TaskExecutorProcessUtilsTest method testProcessSpecFromWorkerResourceSpec.

@Test
public void testProcessSpecFromWorkerResourceSpec() {
    final WorkerResourceSpec workerResourceSpec = new WorkerResourceSpec.Builder().setCpuCores(1.0).setTaskHeapMemoryMB(100).setTaskOffHeapMemoryMB(200).setNetworkMemoryMB(300).setManagedMemoryMB(400).setNumSlots(5).setExtendedResource(new ExternalResource(EXTERNAL_RESOURCE_NAME_1, 1)).build();
    final TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromWorkerResourceSpec(new Configuration(), workerResourceSpec);
    assertEquals(workerResourceSpec.getCpuCores(), taskExecutorProcessSpec.getCpuCores());
    assertEquals(workerResourceSpec.getTaskHeapSize(), taskExecutorProcessSpec.getTaskHeapSize());
    assertEquals(workerResourceSpec.getTaskOffHeapSize(), taskExecutorProcessSpec.getTaskOffHeapSize());
    assertEquals(workerResourceSpec.getNetworkMemSize(), taskExecutorProcessSpec.getNetworkMemSize());
    assertEquals(workerResourceSpec.getManagedMemSize(), taskExecutorProcessSpec.getManagedMemorySize());
    assertEquals(workerResourceSpec.getNumSlots(), taskExecutorProcessSpec.getNumSlots());
    assertEquals(workerResourceSpec.getExtendedResources(), taskExecutorProcessSpec.getExtendedResources());
}