Examples with Priority org.apache.hadoop.yarn.api.records.Priority used on opensource projects

Example 91 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project flink by apache.

the class TaskExecutorProcessSpecContainerResourcePriorityAdapter method tryAdaptAndAddTaskExecutorResourceSpecIfNotExist.

private void tryAdaptAndAddTaskExecutorResourceSpecIfNotExist(final TaskExecutorProcessSpec taskExecutorProcessSpec) {
    if (!taskExecutorProcessSpecToPriorityAndResource.containsKey(taskExecutorProcessSpec)) {
        tryAdaptResource(taskExecutorProcessSpec).ifPresent((resource) -> {
            final Priority priority = Priority.newInstance(nextPriority++);
            taskExecutorProcessSpecToPriorityAndResource.put(taskExecutorProcessSpec, new PriorityAndResource(priority, resource));
            priorityToTaskExecutorProcessSpec.put(priority, taskExecutorProcessSpec);
        });
    }
}

Example 92 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project flink by apache.

the class YarnResourceManagerDriverTest method testStartWorkerVariousSpec.

@Test
public void testStartWorkerVariousSpec() throws Exception {
    final TaskExecutorProcessSpec taskExecutorProcessSpec1 = new TaskExecutorProcessSpec(new CPUResource(1), MemorySize.ZERO, MemorySize.ZERO, MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ZERO, MemorySize.ZERO, Collections.emptyList());
    final TaskExecutorProcessSpec taskExecutorProcessSpec2 = new TaskExecutorProcessSpec(new CPUResource(2), MemorySize.ZERO, MemorySize.ZERO, MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ZERO, MemorySize.ZERO, Collections.emptyList());
    new Context() {

        {
            final String startCommand1 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (50L << 20);
            final String startCommand2 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (100L << 20);
            final CompletableFuture<Void> startContainerAsyncCommandFuture1 = new CompletableFuture<>();
            final CompletableFuture<Void> startContainerAsyncCommandFuture2 = new CompletableFuture<>();
            prepareForTestStartTaskExecutorProcessVariousSpec(startCommand1, startCommand2, startContainerAsyncCommandFuture1, startContainerAsyncCommandFuture2, taskExecutorProcessSpec1);
            testingYarnAMRMClientAsyncBuilder.setGetMatchingRequestsFunction(tuple -> {
                final Priority priority = tuple.f0;
                final List<AMRMClient.ContainerRequest> matchingRequests = new ArrayList<>();
                for (CompletableFuture<AMRMClient.ContainerRequest> addContainerRequestFuture : addContainerRequestFutures) {
                    final AMRMClient.ContainerRequest request = addContainerRequestFuture.getNow(null);
                    if (request != null && priority.equals(request.getPriority())) {
                        assertThat(tuple.f2, is(request.getCapability()));
                        matchingRequests.add(request);
                    }
                }
                return Collections.singletonList(matchingRequests);
            });
            runTest(() -> {
                final Resource containerResource1 = ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec1).get();
                final Resource containerResource2 = ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec2).get();
                // Make sure two worker resource spec will be normalized to different
                // container resources
                assertNotEquals(containerResource1, containerResource2);
                runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec1));
                runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec2));
                // Verify both containers requested
                verifyFutureCompleted(addContainerRequestFutures.get(0));
                verifyFutureCompleted(addContainerRequestFutures.get(1));
                // Mock that container 1 is allocated
                Container container1 = createTestingContainerWithResource(containerResource1);
                resourceManagerClientCallbackHandler.onContainersAllocated(Collections.singletonList(container1));
                // Verify that only worker with spec1 is started.
                verifyFutureCompleted(startContainerAsyncCommandFuture1);
                assertFalse(startContainerAsyncCommandFuture2.isDone());
                // Mock that container 1 is completed, while the worker is still pending
                ContainerStatus testingContainerStatus = createTestingContainerCompletedStatus(container1.getId());
                resourceManagerClientCallbackHandler.onContainersCompleted(Collections.singletonList(testingContainerStatus));
                // Verify that only container 1 is requested again
                verifyFutureCompleted(addContainerRequestFutures.get(2));
                assertThat(addContainerRequestFutures.get(2).get().getCapability(), is(containerResource1));
                assertFalse(addContainerRequestFutures.get(3).isDone());
            });
        }
    };
}

Example 93 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project flink by apache.

the class ContainerRequestReflectorTest method testGetContainerRequestIfConstructorPresent.

@Test
public void testGetContainerRequestIfConstructorPresent() {
    final ContainerRequestReflector containerRequestReflector = new ContainerRequestReflector(ContainerRequestWithConstructor.class);
    Resource resource = Resource.newInstance(100, 1);
    Priority priority = Priority.newInstance(1);
    AMRMClient.ContainerRequest containerRequest = containerRequestReflector.getContainerRequest(resource, priority, "GPU");
    assertTrue(containerRequest instanceof ContainerRequestWithConstructor);
    ContainerRequestWithConstructor containerRequestWithConstructor = (ContainerRequestWithConstructor) containerRequest;
    assertEquals("GPU", containerRequestWithConstructor.getNodeLabelsExpression());
    containerRequest = containerRequestReflector.getContainerRequest(resource, priority, null);
    assertFalse(containerRequest instanceof ContainerRequestWithConstructor);
    containerRequest = containerRequestReflector.getContainerRequest(resource, priority, "");
    assertFalse(containerRequest instanceof ContainerRequestWithConstructor);
}

Example 94 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project kitten by cloudera.

the class ContainerLaunchContextFactory method createPriority.

public Priority createPriority(int priority) {
    Priority p = Records.newRecord(Priority.class);
    p.setPriority(priority);
    return p;
}

Example 95 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hive by apache.

the class LlapTaskSchedulerService method removePendingTask.

/* Remove a task from the pending list */
private void removePendingTask(TaskInfo taskInfo) {
    writeLock.lock();
    try {
        Priority priority = taskInfo.priority;
        List<TaskInfo> taskInfoList = pendingTasks.get(priority);
        if (taskInfoList == null || taskInfoList.isEmpty() || !taskInfoList.remove(taskInfo)) {
            LOG.warn("Could not find task: " + taskInfo.task + " in pending list, at priority: " + priority);
        }
    } finally {
        writeLock.unlock();
    }
}