Examples with ClientRMService org.apache.hadoop.yarn.server.resourcemanager.ClientRMService used on opensource projects

Example 11 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestDelegationTokenRenewer method testConcurrentAddApplication.

@Test(timeout = 20000)
public void testConcurrentAddApplication() throws IOException, InterruptedException, BrokenBarrierException {
    final CyclicBarrier startBarrier = new CyclicBarrier(2);
    final CyclicBarrier endBarrier = new CyclicBarrier(2);
    // this token uses barriers to block during renew                          
    final Credentials creds1 = new Credentials();
    final Token<DelegationTokenIdentifier> token1 = mock(Token.class);
    when(token1.getKind()).thenReturn(KIND);
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"), new Text("user1"));
    when(token1.decodeIdentifier()).thenReturn(dtId1);
    creds1.addToken(new Text("token"), token1);
    doReturn(true).when(token1).isManaged();
    doAnswer(new Answer<Long>() {

        public Long answer(InvocationOnMock invocation) throws InterruptedException, BrokenBarrierException {
            startBarrier.await();
            endBarrier.await();
            return Long.MAX_VALUE;
        }
    }).when(token1).renew(any(Configuration.class));
    // this dummy token fakes renewing                                         
    final Credentials creds2 = new Credentials();
    final Token<DelegationTokenIdentifier> token2 = mock(Token.class);
    when(token2.getKind()).thenReturn(KIND);
    when(token2.decodeIdentifier()).thenReturn(dtId1);
    creds2.addToken(new Text("token"), token2);
    doReturn(true).when(token2).isManaged();
    doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));
    // fire up the renewer                                                     
    final DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter);
    RMContext mockContext = mock(RMContext.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    dtr.setRMContext(mockContext);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
    dtr.init(conf);
    dtr.start();
    // submit a job that blocks during renewal                                 
    Thread submitThread = new Thread() {

        @Override
        public void run() {
            dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user", new Configuration());
        }
    };
    submitThread.start();
    // wait till 1st submit blocks, then submit another
    startBarrier.await();
    dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user", new Configuration());
    // signal 1st to complete                                                  
    endBarrier.await();
    submitThread.join();
}

Example 12 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestDelegationTokenRenewer method testDTKeepAlive1.

/**
   * Basic idea of the test:
   * 0. Setup token KEEP_ALIVE
   * 1. create tokens.
   * 2. register them for renewal - to be cancelled on app complete
   * 3. Complete app.
   * 4. Verify token is alive within the KEEP_ALIVE time
   * 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
   * @throws IOException
   * @throws URISyntaxException
   */
@Test(timeout = 60000)
public void testDTKeepAlive1() throws Exception {
    Configuration lconf = new Configuration(conf);
    lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    //Keep tokens alive for 6 seconds.
    lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
    //Try removing tokens every second.
    lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
    DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter);
    RMContext mockContext = mock(RMContext.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    localDtr.setRMContext(mockContext);
    localDtr.init(lconf);
    localDtr.start();
    MyFS dfs = (MyFS) FileSystem.get(lconf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());
    Credentials ts = new Credentials();
    // get the delegation tokens
    MyToken token1 = dfs.getDelegationToken("user1");
    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);
    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    localDtr.addApplicationAsync(applicationId_0, ts, true, "user", new Configuration());
    waitForEventsToGetProcessed(localDtr);
    if (!eventQueue.isEmpty()) {
        Event evt = eventQueue.take();
        if (evt instanceof RMAppEvent) {
            Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START);
        } else {
            fail("RMAppEvent.START was expected!!");
        }
    }
    localDtr.applicationFinished(applicationId_0);
    waitForEventsToGetProcessed(localDtr);
    //Token should still be around. Renewal should not fail.
    token1.renew(lconf);
    //Allow the keepalive time to run out
    Thread.sleep(10000l);
    //The token should have been cancelled at this point. Renewal will fail.
    try {
        token1.renew(lconf);
        fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
    }
}

Example 13 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestDelegationTokenRenewerLifecycle method testStartupFailure.

@Test
public void testStartupFailure() throws Exception {
    Configuration conf = new Configuration();
    DelegationTokenRenewer delegationTokenRenewer = new DelegationTokenRenewer();
    RMContext mockContext = mock(RMContext.class);
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    delegationTokenRenewer.setRMContext(mockContext);
    delegationTokenRenewer.init(conf);
    delegationTokenRenewer.stop();
}

Example 14 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestRMWebApp method mockRm.

public static ResourceManager mockRm(RMContext rmContext) throws IOException {
    ResourceManager rm = mock(ResourceManager.class);
    ResourceScheduler rs = mockCapacityScheduler();
    ApplicationACLsManager aclMgr = mockAppACLsManager();
    ClientRMService clientRMService = mockClientRMService(rmContext);
    when(rm.getResourceScheduler()).thenReturn(rs);
    when(rm.getRMContext()).thenReturn(rmContext);
    when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
    when(rm.getClientRMService()).thenReturn(clientRMService);
    return rm;
}

Example 15 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project apex-core by apache.

the class StramMiniClusterTest method checkNodeState.

private void checkNodeState() throws YarnException {
    GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
    ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
    GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
    List<NodeReport> nodeReports = response.getNodeReports();
    LOG.info("{}", nodeReports);
    for (NodeReport nr : nodeReports) {
        if (!nr.getNodeState().isUnusable()) {
            return;
        }
    }
    fail("Yarn Mini cluster should have at least one usable node.");
}