Examples with TaskExecutorProcessSpec org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec used on opensource projects

Example 16 with TaskExecutorProcessSpec

use of org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec in project flink by apache.

the class TaskExecutorProcessSpecTest method testEquals.

@Test
public void testEquals() {
    TaskExecutorProcessSpec spec1 = new TaskExecutorProcessSpec(new CPUResource(1.0), MemorySize.parse("1m"), MemorySize.parse("2m"), MemorySize.parse("3m"), MemorySize.parse("4m"), MemorySize.parse("5m"), MemorySize.parse("6m"), MemorySize.parse("7m"), MemorySize.parse("8m"), Collections.singleton(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1)));
    TaskExecutorProcessSpec spec2 = new TaskExecutorProcessSpec(new CPUResource(1.0), MemorySize.parse("1m"), MemorySize.parse("2m"), MemorySize.parse("3m"), MemorySize.parse("4m"), MemorySize.parse("5m"), MemorySize.parse("6m"), MemorySize.parse("7m"), MemorySize.parse("8m"), Collections.singleton(new ExternalResource(EXTERNAL_RESOURCE_NAME, 1)));
    assertThat(spec1, is(spec2));
}

Example 17 with TaskExecutorProcessSpec

use of org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec in project flink by apache.

the class WorkerResourceSpecTest method testCreateFromTaskExecutorProcessSpec.

@Test
public void testCreateFromTaskExecutorProcessSpec() {
    final Configuration config = new Configuration();
    config.setString(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST.key(), EXTERNAL_RESOURCE_NAME);
    config.setLong(ExternalResourceOptions.getAmountConfigOptionForResource(EXTERNAL_RESOURCE_NAME), 1);
    final TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.newProcessSpecBuilder(config).withTotalProcessMemory(MemorySize.ofMebiBytes(1024)).build();
    final WorkerResourceSpec workerResourceSpec = WorkerResourceSpec.fromTaskExecutorProcessSpec(taskExecutorProcessSpec);
    assertEquals(workerResourceSpec.getCpuCores(), taskExecutorProcessSpec.getCpuCores());
    assertEquals(workerResourceSpec.getTaskHeapSize(), taskExecutorProcessSpec.getTaskHeapSize());
    assertEquals(workerResourceSpec.getTaskOffHeapSize(), taskExecutorProcessSpec.getTaskOffHeapSize());
    assertEquals(workerResourceSpec.getNetworkMemSize(), taskExecutorProcessSpec.getNetworkMemSize());
    assertEquals(workerResourceSpec.getManagedMemSize(), taskExecutorProcessSpec.getManagedMemorySize());
    assertEquals(workerResourceSpec.getNumSlots(), taskExecutorProcessSpec.getNumSlots());
    assertEquals(workerResourceSpec.getExtendedResources(), taskExecutorProcessSpec.getExtendedResources());
}

Example 18 with TaskExecutorProcessSpec

use of org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec in project flink by apache.

the class ActiveResourceManagerTest method testStartNewWorker.

/**
 * Tests worker successfully requested, started and registered.
 */
@Test
public void testStartNewWorker() throws Exception {
    new Context() {

        {
            final ResourceID tmResourceId = ResourceID.generate();
            final CompletableFuture<TaskExecutorProcessSpec> requestWorkerFromDriverFuture = new CompletableFuture<>();
            driverBuilder.setRequestResourceFunction(taskExecutorProcessSpec -> {
                requestWorkerFromDriverFuture.complete(taskExecutorProcessSpec);
                return CompletableFuture.completedFuture(tmResourceId);
            });
            runTest(() -> {
                // received worker request, verify requesting from driver
                CompletableFuture<Boolean> startNewWorkerFuture = runInMainThread(() -> getResourceManager().startNewWorker(WORKER_RESOURCE_SPEC));
                TaskExecutorProcessSpec taskExecutorProcessSpec = requestWorkerFromDriverFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS);
                assertThat(startNewWorkerFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS), is(true));
                assertThat(taskExecutorProcessSpec, is(TaskExecutorProcessUtils.processSpecFromWorkerResourceSpec(flinkConfig, WORKER_RESOURCE_SPEC)));
                // worker registered, verify registration succeeded
                CompletableFuture<RegistrationResponse> registerTaskExecutorFuture = registerTaskExecutor(tmResourceId);
                assertThat(registerTaskExecutorFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS), instanceOf(RegistrationResponse.Success.class));
            });
        }
    };
}

Example 19 with TaskExecutorProcessSpec

use of org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec in project flink by apache.

the class ActiveResourceManagerTest method testStartNewWorkerFailedRequesting.

/**
 * Tests worker failed while requesting.
 */
@Test
public void testStartNewWorkerFailedRequesting() throws Exception {
    new Context() {

        {
            final ResourceID tmResourceId = ResourceID.generate();
            final AtomicInteger requestCount = new AtomicInteger(0);
            final List<CompletableFuture<ResourceID>> resourceIdFutures = new ArrayList<>();
            resourceIdFutures.add(new CompletableFuture<>());
            resourceIdFutures.add(new CompletableFuture<>());
            final List<CompletableFuture<TaskExecutorProcessSpec>> requestWorkerFromDriverFutures = new ArrayList<>();
            requestWorkerFromDriverFutures.add(new CompletableFuture<>());
            requestWorkerFromDriverFutures.add(new CompletableFuture<>());
            driverBuilder.setRequestResourceFunction(taskExecutorProcessSpec -> {
                int idx = requestCount.getAndIncrement();
                assertThat(idx, lessThan(2));
                requestWorkerFromDriverFutures.get(idx).complete(taskExecutorProcessSpec);
                return resourceIdFutures.get(idx);
            });
            slotManagerBuilder.setGetRequiredResourcesSupplier(() -> Collections.singletonMap(WORKER_RESOURCE_SPEC, 1));
            runTest(() -> {
                // received worker request, verify requesting from driver
                CompletableFuture<Boolean> startNewWorkerFuture = runInMainThread(() -> getResourceManager().startNewWorker(WORKER_RESOURCE_SPEC));
                TaskExecutorProcessSpec taskExecutorProcessSpec1 = requestWorkerFromDriverFutures.get(0).get(TIMEOUT_SEC, TimeUnit.SECONDS);
                assertThat(startNewWorkerFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS), is(true));
                assertThat(taskExecutorProcessSpec1, is(TaskExecutorProcessUtils.processSpecFromWorkerResourceSpec(flinkConfig, WORKER_RESOURCE_SPEC)));
                // first request failed, verify requesting another worker from driver
                runInMainThread(() -> resourceIdFutures.get(0).completeExceptionally(new Throwable("testing error")));
                TaskExecutorProcessSpec taskExecutorProcessSpec2 = requestWorkerFromDriverFutures.get(1).get(TIMEOUT_SEC, TimeUnit.SECONDS);
                assertThat(taskExecutorProcessSpec2, is(taskExecutorProcessSpec1));
                // second request allocated, verify registration succeed
                runInMainThread(() -> resourceIdFutures.get(1).complete(tmResourceId));
                CompletableFuture<RegistrationResponse> registerTaskExecutorFuture = registerTaskExecutor(tmResourceId);
                assertThat(registerTaskExecutorFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS), instanceOf(RegistrationResponse.Success.class));
            });
        }
    };
}

Example 20 with TaskExecutorProcessSpec

use of org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec in project flink by apache.

the class ActiveResourceManagerTest method testWorkerTerminatedAfterRegister.

/**
 * Tests worker terminated after registered.
 */
@Test
public void testWorkerTerminatedAfterRegister() throws Exception {
    new Context() {

        {
            final AtomicInteger requestCount = new AtomicInteger(0);
            final List<ResourceID> tmResourceIds = new ArrayList<>();
            tmResourceIds.add(ResourceID.generate());
            tmResourceIds.add(ResourceID.generate());
            final List<CompletableFuture<TaskExecutorProcessSpec>> requestWorkerFromDriverFutures = new ArrayList<>();
            requestWorkerFromDriverFutures.add(new CompletableFuture<>());
            requestWorkerFromDriverFutures.add(new CompletableFuture<>());
            driverBuilder.setRequestResourceFunction(taskExecutorProcessSpec -> {
                int idx = requestCount.getAndIncrement();
                assertThat(idx, lessThan(2));
                requestWorkerFromDriverFutures.get(idx).complete(taskExecutorProcessSpec);
                return CompletableFuture.completedFuture(tmResourceIds.get(idx));
            });
            slotManagerBuilder.setGetRequiredResourcesSupplier(() -> Collections.singletonMap(WORKER_RESOURCE_SPEC, 1));
            runTest(() -> {
                // received worker request, verify requesting from driver
                CompletableFuture<Boolean> startNewWorkerFuture = runInMainThread(() -> getResourceManager().startNewWorker(WORKER_RESOURCE_SPEC));
                TaskExecutorProcessSpec taskExecutorProcessSpec1 = requestWorkerFromDriverFutures.get(0).get(TIMEOUT_SEC, TimeUnit.SECONDS);
                assertThat(startNewWorkerFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS), is(true));
                assertThat(taskExecutorProcessSpec1, is(TaskExecutorProcessUtils.processSpecFromWorkerResourceSpec(flinkConfig, WORKER_RESOURCE_SPEC)));
                // first worker registered, verify registration succeed
                CompletableFuture<RegistrationResponse> registerTaskExecutorFuture1 = registerTaskExecutor(tmResourceIds.get(0));
                assertThat(registerTaskExecutorFuture1.get(TIMEOUT_SEC, TimeUnit.SECONDS), instanceOf(RegistrationResponse.Success.class));
                // first worker terminated, verify requesting another worker from driver
                runInMainThread(() -> getResourceManager().onWorkerTerminated(tmResourceIds.get(0), "terminate for testing"));
                TaskExecutorProcessSpec taskExecutorProcessSpec2 = requestWorkerFromDriverFutures.get(1).get(TIMEOUT_SEC, TimeUnit.SECONDS);
                assertThat(taskExecutorProcessSpec2, is(taskExecutorProcessSpec1));
                // second worker registered, verify registration succeed
                CompletableFuture<RegistrationResponse> registerTaskExecutorFuture2 = registerTaskExecutor(tmResourceIds.get(1));
                assertThat(registerTaskExecutorFuture2.get(TIMEOUT_SEC, TimeUnit.SECONDS), instanceOf(RegistrationResponse.Success.class));
            });
        }
    };
}