Examples with AllocateRequest org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest used on opensource projects

Example 16 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestCapacityScheduler method testAllocateDoesNotBlockOnSchedulerLock.

@Test(timeout = 30000)
public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception {
    final YarnConfiguration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    MockNM nm1 = rm.registerNode("localhost:1234", 5120);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(2);
    acls.put(ApplicationAccessType.VIEW_APP, "*");
    RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
    int msecToWait = 10000;
    int msecToSleep = 100;
    while (attempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED && msecToWait > 0) {
        LOG.info("Waiting for AppAttempt to reach LAUNCHED state. " + "Current state is " + attempt.getAppAttemptState());
        Thread.sleep(msecToSleep);
        msecToWait -= msecToSleep;
    }
    Assert.assertEquals(attempt.getAppAttemptState(), RMAppAttemptState.LAUNCHED);
    // Create a client to the RM.
    final YarnRPC rpc = YarnRPC.create(conf);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
    Credentials credentials = containerManager.getContainerCredentials();
    final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
    Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);
    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
    });
    RegisterApplicationMasterRequest request = RegisterApplicationMasterRequest.newInstance("localhost", 12345, "");
    client.registerApplicationMaster(request);
    // Allocate a container
    List<ResourceRequest> asks = Collections.singletonList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(2 * GB), 1));
    AllocateRequest allocateRequest = AllocateRequest.newInstance(0, 0.0f, asks, null, null);
    client.allocate(allocateRequest);
    // Make sure the container is allocated in RM
    nm1.nodeHeartbeat(true);
    ContainerId containerId2 = ContainerId.newContainerId(applicationAttemptId, 2);
    Assert.assertTrue(rm.waitForState(nm1, containerId2, RMContainerState.ALLOCATED));
    // Acquire the container
    allocateRequest = AllocateRequest.newInstance(1, 0.0f, null, null, null);
    client.allocate(allocateRequest);
    // Launch the container
    final CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    RMContainer rmContainer = cs.getRMContainer(containerId2);
    rmContainer.handle(new RMContainerEvent(containerId2, RMContainerEventType.LAUNCHED));
    // grab the scheduler lock from another thread
    // and verify an allocate call in this thread doesn't block on it
    final CyclicBarrier barrier = new CyclicBarrier(2);
    Thread otherThread = new Thread(new Runnable() {

        @Override
        public void run() {
            synchronized (cs) {
                try {
                    barrier.await();
                    barrier.await();
                } catch (InterruptedException | BrokenBarrierException e) {
                    e.printStackTrace();
                }
            }
        }
    });
    otherThread.start();
    barrier.await();
    List<ContainerId> release = Collections.singletonList(containerId2);
    allocateRequest = AllocateRequest.newInstance(2, 0.0f, null, release, null);
    client.allocate(allocateRequest);
    barrier.await();
    otherThread.join();
    rm.stop();
}

Example 17 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestOpportunisticContainerAllocatorAMService method testRPCWrapping.

// Test if the OpportunisticContainerAllocatorAMService can handle both
// DSProtocol as well as AMProtocol clients
@Test
public void testRPCWrapping() throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, addr);
    final RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
    final RMContext rmContext = new RMContextImpl() {

        @Override
        public AMLivelinessMonitor getAMLivelinessMonitor() {
            return null;
        }

        @Override
        public Configuration getYarnConfiguration() {
            return new YarnConfiguration();
        }

        @Override
        public RMContainerTokenSecretManager getContainerTokenSecretManager() {
            return new RMContainerTokenSecretManager(conf);
        }
    };
    Container c = factory.newRecordInstance(Container.class);
    c.setExecutionType(ExecutionType.OPPORTUNISTIC);
    c.setId(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(12345, 1), 2), 3));
    AllocateRequest allReq = (AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class);
    allReq.setAskList(Arrays.asList(ResourceRequest.newInstance(Priority.UNDEFINED, "a", Resource.newInstance(1, 2), 1, true, "exp", ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true))));
    OpportunisticContainerAllocatorAMService service = createService(factory, rmContext, c);
    conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
    Server server = service.getServer(rpc, conf, addr, null);
    server.start();
    // Verify that the OpportunisticContainerAllocatorAMSercvice can handle
    // vanilla ApplicationMasterProtocol clients
    RPC.setProtocolEngine(conf, ApplicationMasterProtocolPB.class, ProtobufRpcEngine.class);
    ApplicationMasterProtocolPB ampProxy = RPC.getProxy(ApplicationMasterProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
    RegisterApplicationMasterResponse regResp = new RegisterApplicationMasterResponsePBImpl(ampProxy.registerApplicationMaster(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals("dummyQueue", regResp.getQueue());
    FinishApplicationMasterResponse finishResp = new FinishApplicationMasterResponsePBImpl(ampProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(false, finishResp.getIsUnregistered());
    AllocateResponse allocResp = new AllocateResponsePBImpl(ampProxy.allocate(null, ((AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class)).getProto()));
    List<Container> allocatedContainers = allocResp.getAllocatedContainers();
    Assert.assertEquals(1, allocatedContainers.size());
    Assert.assertEquals(ExecutionType.OPPORTUNISTIC, allocatedContainers.get(0).getExecutionType());
    Assert.assertEquals(12345, allocResp.getNumClusterNodes());
    // Verify that the DistrubutedSchedulingService can handle the
    // DistributedSchedulingAMProtocol clients as well
    RPC.setProtocolEngine(conf, DistributedSchedulingAMProtocolPB.class, ProtobufRpcEngine.class);
    DistributedSchedulingAMProtocolPB dsProxy = RPC.getProxy(DistributedSchedulingAMProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
    RegisterDistributedSchedulingAMResponse dsRegResp = new RegisterDistributedSchedulingAMResponsePBImpl(dsProxy.registerApplicationMasterForDistributedScheduling(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(54321l, dsRegResp.getContainerIdStart());
    Assert.assertEquals(4, dsRegResp.getMaxContainerResource().getVirtualCores());
    Assert.assertEquals(1024, dsRegResp.getMinContainerResource().getMemorySize());
    Assert.assertEquals(2, dsRegResp.getIncrContainerResource().getVirtualCores());
    DistributedSchedulingAllocateRequestPBImpl distAllReq = (DistributedSchedulingAllocateRequestPBImpl) factory.newRecordInstance(DistributedSchedulingAllocateRequest.class);
    distAllReq.setAllocateRequest(allReq);
    distAllReq.setAllocatedContainers(Arrays.asList(c));
    DistributedSchedulingAllocateResponse dsAllocResp = new DistributedSchedulingAllocateResponsePBImpl(dsProxy.allocateForDistributedScheduling(null, distAllReq.getProto()));
    Assert.assertEquals("h1", dsAllocResp.getNodesForScheduling().get(0).getNodeId().getHost());
    FinishApplicationMasterResponse dsfinishResp = new FinishApplicationMasterResponsePBImpl(dsProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(false, dsfinishResp.getIsUnregistered());
}

Example 18 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMTokens method testTokenExpiry.

/**
   * Validate that application tokens are unusable after the
   * application-finishes.
   * 
   * @throws Exception
   */
@SuppressWarnings("unchecked")
@Test
public void testTokenExpiry() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:0");
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    ApplicationMasterProtocol rmClient = null;
    try {
        MockNM nm1 = rm.registerNode("localhost:1234", 5120);
        RMApp app = rm.submitApp(1024);
        nm1.nodeHeartbeat(true);
        int waitCount = 0;
        while (containerManager.containerTokens == null && waitCount++ < 20) {
            LOG.info("Waiting for AM Launch to happen..");
            Thread.sleep(1000);
        }
        Assert.assertNotNull(containerManager.containerTokens);
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
        // Create a client to the RM.
        UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
        Credentials credentials = containerManager.getContainerCredentials();
        final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
        Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
        currentUser.addToken(amRMToken);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
        rmClient.registerApplicationMaster(request);
        FinishApplicationMasterRequest finishAMRequest = Records.newRecord(FinishApplicationMasterRequest.class);
        finishAMRequest.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
        finishAMRequest.setDiagnostics("diagnostics");
        finishAMRequest.setTrackingUrl("url");
        rmClient.finishApplicationMaster(finishAMRequest);
        // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt
        // from Finishing state to Finished State. Both AMRMToken and
        // ClientToAMToken will be removed.
        ContainerStatus containerStatus = BuilderUtils.newContainerStatus(attempt.getMasterContainer().getId(), ContainerState.COMPLETE, "AM Container Finished", 0, attempt.getMasterContainer().getResource());
        rm.getRMContext().getDispatcher().getEventHandler().handle(new RMAppAttemptContainerFinishedEvent(applicationAttemptId, containerStatus, nm1.getNodeId()));
        // Make sure the RMAppAttempt is at Finished State.
        // Both AMRMToken and ClientToAMToken have been removed.
        int count = 0;
        while (attempt.getState() != RMAppAttemptState.FINISHED && count < maxWaitAttempts) {
            Thread.sleep(100);
            count++;
        }
        Assert.assertTrue(attempt.getState() == RMAppAttemptState.FINISHED);
        // Now simulate trying to allocate. RPC call itself should throw auth
        // exception.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
        try {
            rmClient.allocate(allocateRequest);
            Assert.fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!");
        } catch (Throwable t) {
            LOG.info("Exception found is ", t);
            // The exception will still have the earlier appAttemptId as it picks it
            // up from the token.
            Assert.assertTrue(t.getCause().getMessage().contains(applicationAttemptId.toString() + " not found in AMRMTokenSecretManager."));
        }
    } finally {
        rm.stop();
        if (rmClient != null) {
            // To avoid using cached client
            rpc.stopProxy(rmClient, conf);
        }
    }
}

Example 19 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project weave by continuuity.

the class AMRMClientImpl method allocate.

@Override
public AllocationResponse allocate(float progressIndicator) throws YarnRemoteException {
    AllocateResponse allocateResponse = null;
    ArrayList<ResourceRequest> askList = null;
    ArrayList<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    try {
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask);
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            allocateRequest = BuilderUtils.newAllocateRequest(appAttemptId, lastResponseId, progressIndicator, askList, releaseList);
        }
        allocateResponse = rmClient.allocate(allocateRequest);
        AllocationResponse response = AllocationResponses.create(allocateResponse);
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = response.getResponseId();
            clusterAvailableResources = response.getAvailableResources();
        }
        return response;
    } finally {
        // TODO how to differentiate remote YARN exception vs error in RPC
        if (allocateResponse == null) {
            // Preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
            }
        }
    }
}

Example 20 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class LocalContainerAllocator method heartbeat.

@SuppressWarnings("unchecked")
@Override
protected synchronized void heartbeat() throws Exception {
    AllocateRequest allocateRequest = AllocateRequest.newInstance(this.lastResponseID, super.getApplicationProgress(), new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), null);
    AllocateResponse allocateResponse = null;
    try {
        allocateResponse = scheduler.allocate(allocateRequest);
        // Reset retry count if no exception occurred.
        retrystartTime = System.currentTimeMillis();
    } catch (ApplicationAttemptNotFoundException e) {
        LOG.info("Event from RM: shutting down Application Master");
        // This can happen if the RM has been restarted. If it is in that state,
        // this application must clean itself up.
        eventHandler.handle(new JobEvent(this.getJob().getID(), JobEventType.JOB_AM_REBOOT));
        throw new YarnRuntimeException("Resource Manager doesn't recognize AttemptId: " + this.getContext().getApplicationID(), e);
    } catch (ApplicationMasterNotRegisteredException e) {
        LOG.info("ApplicationMaster is out of sync with ResourceManager," + " hence resync and send outstanding requests.");
        this.lastResponseID = 0;
        register();
    } catch (Exception e) {
        // re-trying until the retryInterval has expired.
        if (System.currentTimeMillis() - retrystartTime >= retryInterval) {
            LOG.error("Could not contact RM after " + retryInterval + " milliseconds.");
            eventHandler.handle(new JobEvent(this.getJob().getID(), JobEventType.INTERNAL_ERROR));
            throw new YarnRuntimeException("Could not contact RM after " + retryInterval + " milliseconds.");
        }
        // continue to attempt to contact the RM.
        throw e;
    }
    if (allocateResponse != null) {
        this.lastResponseID = allocateResponse.getResponseId();
        Token token = allocateResponse.getAMRMToken();
        if (token != null) {
            updateAMRMToken(token);
        }
        Priority priorityFromResponse = Priority.newInstance(allocateResponse.getApplicationPriority().getPriority());
        // Update the job priority to Job directly.
        getJob().setJobPriority(priorityFromResponse);
    }
}