Examples with Priority org.apache.hadoop.yarn.api.records.Priority used on opensource projects

Example 46 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestOpportunisticContainerAllocation method startApp.

@Before
public void startApp() throws Exception {
    // submit new app
    ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    // set the application name
    appContext.setApplicationName("Test");
    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(0);
    appContext.setPriority(pri);
    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("default");
    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext(Collections.<String, LocalResource>emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), null, new HashMap<ApplicationAccessType, String>());
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // Create the request to send to the applications manager
    SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class);
    appRequest.setApplicationSubmissionContext(appContext);
    // Submit the application to the applications manager
    yarnClient.submitApplication(appContext);
    // wait for app to start
    RMAppAttempt appAttempt = null;
    while (true) {
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
            attemptId = appReport.getCurrentApplicationAttemptId();
            appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps().get(attemptId.getApplicationId()).getCurrentAppAttempt();
            while (true) {
                if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
                    break;
                }
            }
            break;
        }
    }
    // Just dig into the ResourceManager and get the AMRMToken just for the sake
    // of testing.
    UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
    // emulate RM setup of AMRM token in credentials by adding the token
    // *before* setting the token service
    UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
    appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
    // start am rm client
    amClient = (AMRMClientImpl<AMRMClient.ContainerRequest>) AMRMClient.createAMRMClient();
    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), amClient.getNMTokenCache());
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
}

Example 47 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestDistributedScheduling method testAMOpportunistic.

/**
   * Check if an AM can ask for opportunistic containers and get them.
   * @throws Exception
   */
@Test
public void testAMOpportunistic() throws Exception {
    // Basic container to request
    Resource capability = Resource.newInstance(1024, 1);
    Priority priority = Priority.newInstance(1);
    // Get the cluster topology
    List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
    String node = nodeReports.get(0).getNodeId().getHost();
    String rack = nodeReports.get(0).getRackName();
    String[] nodes = new String[] { node };
    String[] racks = new String[] { rack };
    // Create an AM to request resources
    AMRMClient<AMRMClient.ContainerRequest> amClient = null;
    try {
        amClient = new AMRMClientImpl<AMRMClient.ContainerRequest>(client);
        amClient.init(yarnConf);
        amClient.start();
        amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
        // AM requests an opportunistic container
        ExecutionTypeRequest execTypeRequest = ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true);
        ContainerRequest containerRequest = new AMRMClient.ContainerRequest(capability, nodes, racks, priority, 0, true, null, execTypeRequest);
        amClient.addContainerRequest(containerRequest);
        // Wait until the container is allocated
        ContainerId opportunisticContainerId = null;
        for (int i = 0; i < 10 && opportunisticContainerId == null; i++) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            List<Container> allocatedContainers = allocResponse.getAllocatedContainers();
            for (Container allocatedContainer : allocatedContainers) {
                // Check that this is the container we required
                assertEquals(ExecutionType.OPPORTUNISTIC, allocatedContainer.getExecutionType());
                opportunisticContainerId = allocatedContainer.getId();
            }
            sleep(100);
        }
        assertNotNull(opportunisticContainerId);
        // The RM sees the container as OPPORTUNISTIC
        ResourceScheduler scheduler = cluster.getResourceManager().getResourceScheduler();
        RMContainer rmContainer = scheduler.getRMContainer(opportunisticContainerId);
        assertEquals(ExecutionType.OPPORTUNISTIC, rmContainer.getExecutionType());
        // Release the opportunistic container
        amClient.releaseAssignedContainer(opportunisticContainerId);
        // Wait for the release container to appear
        boolean released = false;
        for (int i = 0; i < 10 && !released; i++) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            List<ContainerStatus> completedContainers = allocResponse.getCompletedContainersStatuses();
            for (ContainerStatus completedContainer : completedContainers) {
                ContainerId completedContainerId = completedContainer.getContainerId();
                assertEquals(completedContainerId, opportunisticContainerId);
                released = true;
            }
            if (!released) {
                sleep(100);
            }
        }
        assertTrue(released);
        // The RM shouldn't see the container anymore
        rmContainer = scheduler.getRMContainer(opportunisticContainerId);
        assertNull(rmContainer);
        // Clean the AM
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.close();
        }
    }
}

Example 48 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestDistributedScheduling method testAMRMClient.

/**
   * Validates if AMRMClient can be used with Distributed Scheduling turned on.
   *
   * @throws Exception
   */
@Test(timeout = 120000)
@SuppressWarnings("unchecked")
public void testAMRMClient() throws Exception {
    AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
    try {
        Priority priority = Priority.newInstance(1);
        Priority priority2 = Priority.newInstance(2);
        Resource capability = Resource.newInstance(1024, 1);
        List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
        String node = nodeReports.get(0).getNodeId().getHost();
        String rack = nodeReports.get(0).getRackName();
        String[] nodes = new String[] { node };
        String[] racks = new String[] { rack };
        // start am rm client
        amClient = new AMRMClientImpl(client);
        amClient.init(yarnConf);
        amClient.start();
        amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        RemoteRequestsTable<ContainerRequest> remoteRequestsTable = amClient.getTable(0);
        int containersRequestedNode = remoteRequestsTable.get(priority, node, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedRack = remoteRequestsTable.get(priority, rack, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedAny = remoteRequestsTable.get(priority, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int oppContainersRequestedAny = remoteRequestsTable.get(priority2, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
        assertEquals(2, containersRequestedNode);
        assertEquals(2, containersRequestedRack);
        assertEquals(2, containersRequestedAny);
        assertEquals(1, oppContainersRequestedAny);
        assertEquals(4, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        // RM should allocate container within 2 calls to allocate()
        int allocatedContainerCount = 0;
        int iterationsLeft = 10;
        Set<ContainerId> releases = new TreeSet<>();
        amClient.getNMTokenCache().clearCache();
        Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
        HashMap<String, Token> receivedNMTokens = new HashMap<>();
        while (allocatedContainerCount < (containersRequestedAny + oppContainersRequestedAny) && iterationsLeft-- > 0) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            assertEquals(0, amClient.ask.size());
            assertEquals(0, amClient.release.size());
            allocatedContainerCount += allocResponse.getAllocatedContainers().size();
            for (Container container : allocResponse.getAllocatedContainers()) {
                ContainerId rejectContainerId = container.getId();
                releases.add(rejectContainerId);
            }
            for (NMToken token : allocResponse.getNMTokens()) {
                String nodeID = token.getNodeId().toString();
                receivedNMTokens.put(nodeID, token.getToken());
            }
            if (allocatedContainerCount < containersRequestedAny) {
                // sleep to let NM's heartbeat to RM and trigger allocations
                sleep(100);
            }
        }
        assertEquals(allocatedContainerCount, containersRequestedAny + oppContainersRequestedAny);
        for (ContainerId rejectContainerId : releases) {
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        assertEquals(3, amClient.release.size());
        assertEquals(0, amClient.ask.size());
        // need to tell the AMRMClient that we dont need these resources anymore
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        assertEquals(4, amClient.ask.size());
        // test RPC exception handling
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        final AMRMClient amc = amClient;
        ApplicationMasterProtocol realRM = amClient.rmClient;
        try {
            ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
            when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(new Answer<AllocateResponse>() {

                public AllocateResponse answer(InvocationOnMock invocation) throws Exception {
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
                    throw new Exception();
                }
            });
            amClient.rmClient = mockRM;
            amClient.allocate(0.1f);
        } catch (Exception ioe) {
        } finally {
            amClient.rmClient = realRM;
        }
        assertEquals(3, amClient.release.size());
        assertEquals(6, amClient.ask.size());
        iterationsLeft = 3;
        // do a few iterations to ensure RM is not going send new containers
        while (iterationsLeft-- > 0) {
            // inform RM of rejection
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            // RM did not send new containers because AM does not need any
            assertEquals(0, allocResponse.getAllocatedContainers().size());
            if (allocResponse.getCompletedContainersStatuses().size() > 0) {
                for (ContainerStatus cStatus : allocResponse.getCompletedContainersStatuses()) {
                    if (releases.contains(cStatus.getContainerId())) {
                        assertEquals(cStatus.getState(), ContainerState.COMPLETE);
                        assertEquals(-100, cStatus.getExitStatus());
                        releases.remove(cStatus.getContainerId());
                    }
                }
            }
            if (iterationsLeft > 0) {
                // sleep to make sure NM's heartbeat
                sleep(100);
            }
        }
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 49 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class ClientRMService method updateApplicationPriority.

@Override
public UpdateApplicationPriorityResponse updateApplicationPriority(UpdateApplicationPriorityRequest request) throws YarnException, IOException {
    ApplicationId applicationId = request.getApplicationId();
    Priority newAppPriority = request.getApplicationPriority();
    UserGroupInformation callerUGI = getCallerUgi(applicationId, AuditConstants.UPDATE_APP_PRIORITY);
    RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, AuditConstants.UPDATE_APP_PRIORITY);
    UpdateApplicationPriorityResponse response = recordFactory.newRecordInstance(UpdateApplicationPriorityResponse.class);
    // Update priority only when app is tracked by the scheduler
    if (!ACTIVE_APP_STATES.contains(application.getState())) {
        if (application.isAppInCompletedStates()) {
            // If Application is in any of the final states, change priority
            // can be skipped rather throwing exception.
            RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService", applicationId);
            response.setApplicationPriority(application.getApplicationPriority());
            return response;
        }
        String msg = "Application in " + application.getState() + " state cannot update priority.";
        RMAuditLogger.logFailure(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "UNKNOWN", "ClientRMService", msg);
        throw new YarnException(msg);
    }
    try {
        rmAppManager.updateApplicationPriority(callerUGI, application.getApplicationId(), newAppPriority);
    } catch (YarnException ex) {
        RMAuditLogger.logFailure(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "UNKNOWN", "ClientRMService", ex.getMessage());
        throw ex;
    }
    RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService", applicationId);
    response.setApplicationPriority(application.getApplicationPriority());
    return response;
}

Example 50 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class RMAppManager method createAndPopulateNewRMApp.

private RMAppImpl createAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext, long submitTime, String user, boolean isRecovery, long startTime) throws YarnException {
    if (!isRecovery) {
        // Do queue mapping
        if (rmContext.getQueuePlacementManager() != null) {
            // We only do queue mapping when it's a new application
            rmContext.getQueuePlacementManager().placeApplication(submissionContext, user);
        }
        // fail the submission if configured application timeout value is invalid
        RMServerUtils.validateApplicationTimeouts(submissionContext.getApplicationTimeouts());
    }
    ApplicationId applicationId = submissionContext.getApplicationId();
    ResourceRequest amReq = null;
    try {
        amReq = validateAndCreateResourceRequest(submissionContext, isRecovery);
    } catch (InvalidLabelResourceRequestException e) {
        // after recovery and user can see what's going on and react accordingly.
        if (isRecovery && !YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("AMResourceRequest is not created for " + applicationId + ". NodeLabel is not enabled in cluster, but AM resource " + "request contains a label expression.");
            }
        } else {
            throw e;
        }
    }
    // Verify and get the update application priority and set back to
    // submissionContext
    UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
    Priority appPriority = scheduler.checkAndGetApplicationPriority(submissionContext.getPriority(), userUgi, submissionContext.getQueue(), applicationId);
    submissionContext.setPriority(appPriority);
    // For now, exclude FS for the acl check.
    if (!isRecovery && YarnConfiguration.isAclEnabled(conf) && scheduler instanceof CapacityScheduler) {
        String queueName = submissionContext.getQueue();
        String appName = submissionContext.getApplicationName();
        CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName);
        if (null != csqueue && !authorizer.checkPermission(new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS), applicationId.toString(), appName, Server.getRemoteAddress(), null)) && !authorizer.checkPermission(new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE), applicationId.toString(), appName, Server.getRemoteAddress(), null))) {
            throw RPCUtil.getRemoteException(new AccessControlException("User " + user + " does not have permission to submit " + applicationId + " to queue " + submissionContext.getQueue()));
        }
    }
    // Create RMApp
    RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), submissionContext.getApplicationTags(), amReq, startTime);
    // influence each other
    if (rmContext.getRMApps().putIfAbsent(applicationId, application) != null) {
        String message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
        LOG.warn(message);
        throw new YarnException(message);
    }
    if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
        // Start timeline collector for the submitted app
        application.startTimelineCollector();
    }
    // Inform the ACLs Manager
    this.applicationACLsManager.addApplication(applicationId, submissionContext.getAMContainerSpec().getApplicationACLs());
    String appViewACLs = submissionContext.getAMContainerSpec().getApplicationACLs().get(ApplicationAccessType.VIEW_APP);
    rmContext.getSystemMetricsPublisher().appACLsUpdated(application, appViewACLs, System.currentTimeMillis());
    return application;
}