Examples with RMContext org.apache.hadoop.yarn.server.resourcemanager.RMContext used on opensource projects

Example 56 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestRMWebApp method mockFifoRm.

public static ResourceManager mockFifoRm(int apps, int racks, int nodes, int mbsPerNode) throws Exception {
    ResourceManager rm = mock(ResourceManager.class);
    RMContext rmContext = mockRMContext(apps, racks, nodes, mbsPerNode);
    ResourceScheduler rs = mockFifoScheduler(rmContext);
    when(rm.getResourceScheduler()).thenReturn(rs);
    when(rm.getRMContext()).thenReturn(rmContext);
    return rm;
}

Example 57 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestDelegationTokenRenewer method testDTKeepAlive2.

/**
   * Basic idea of the test:
   * 0. Setup token KEEP_ALIVE
   * 1. create tokens.
   * 2. register them for renewal - to be cancelled on app complete
   * 3. Complete app.
   * 4. Verify token is alive within the KEEP_ALIVE time
   * 5. Send an explicity KEEP_ALIVE_REQUEST
   * 6. Verify token KEEP_ALIVE time is renewed.
   * 7. Verify token has been cancelled after the renewed KEEP_ALIVE_TIME.
   * @throws IOException
   * @throws URISyntaxException
   */
@Test(timeout = 60000)
public void testDTKeepAlive2() throws Exception {
    Configuration lconf = new Configuration(conf);
    lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    //Keep tokens alive for 6 seconds.
    lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
    //Try removing tokens every second.
    lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
    DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(conf, counter);
    RMContext mockContext = mock(RMContext.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    localDtr.setRMContext(mockContext);
    localDtr.init(lconf);
    localDtr.start();
    MyFS dfs = (MyFS) FileSystem.get(lconf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());
    Credentials ts = new Credentials();
    // get the delegation tokens
    MyToken token1 = dfs.getDelegationToken("user1");
    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);
    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    localDtr.addApplicationAsync(applicationId_0, ts, true, "user", new Configuration());
    localDtr.applicationFinished(applicationId_0);
    waitForEventsToGetProcessed(delegationTokenRenewer);
    //Send another keep alive.
    localDtr.updateKeepAliveApplications(Collections.singletonList(applicationId_0));
    //Renewal should not fail.
    token1.renew(lconf);
    //Token should be around after this. 
    Thread.sleep(4500l);
    //Renewal should not fail. - ~1.5 seconds for keepalive timeout.
    token1.renew(lconf);
    //Allow the keepalive time to run out
    Thread.sleep(3000l);
    //The token should have been cancelled at this point. Renewal will fail.
    try {
        token1.renew(lconf);
        fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
    }
}

Example 58 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestDelegationTokenRenewer method testDTRonAppSubmission.

@Test(timeout = 20000)
public void testDTRonAppSubmission() throws IOException, InterruptedException, BrokenBarrierException {
    final Credentials credsx = new Credentials();
    final Token<DelegationTokenIdentifier> tokenx = mock(Token.class);
    when(tokenx.getKind()).thenReturn(KIND);
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"), new Text("user1"));
    when(tokenx.decodeIdentifier()).thenReturn(dtId1);
    credsx.addToken(new Text("token"), tokenx);
    doReturn(true).when(tokenx).isManaged();
    doThrow(new IOException("boom")).when(tokenx).renew(any(Configuration.class));
    // fire up the renewer
    final DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter);
    RMContext mockContext = mock(RMContext.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    dtr.setRMContext(mockContext);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
    dtr.init(conf);
    dtr.start();
    try {
        dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user");
        fail("Catch IOException on app submission");
    } catch (IOException e) {
        Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
        Assert.assertTrue(e.getCause().toString().contains("boom"));
    }
}

Example 59 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestDelegationTokenRenewer method testConcurrentAddApplication.

@Test(timeout = 20000)
public void testConcurrentAddApplication() throws IOException, InterruptedException, BrokenBarrierException {
    final CyclicBarrier startBarrier = new CyclicBarrier(2);
    final CyclicBarrier endBarrier = new CyclicBarrier(2);
    // this token uses barriers to block during renew                          
    final Credentials creds1 = new Credentials();
    final Token<DelegationTokenIdentifier> token1 = mock(Token.class);
    when(token1.getKind()).thenReturn(KIND);
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"), new Text("user1"));
    when(token1.decodeIdentifier()).thenReturn(dtId1);
    creds1.addToken(new Text("token"), token1);
    doReturn(true).when(token1).isManaged();
    doAnswer(new Answer<Long>() {

        public Long answer(InvocationOnMock invocation) throws InterruptedException, BrokenBarrierException {
            startBarrier.await();
            endBarrier.await();
            return Long.MAX_VALUE;
        }
    }).when(token1).renew(any(Configuration.class));
    // this dummy token fakes renewing                                         
    final Credentials creds2 = new Credentials();
    final Token<DelegationTokenIdentifier> token2 = mock(Token.class);
    when(token2.getKind()).thenReturn(KIND);
    when(token2.decodeIdentifier()).thenReturn(dtId1);
    creds2.addToken(new Text("token"), token2);
    doReturn(true).when(token2).isManaged();
    doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));
    // fire up the renewer                                                     
    final DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter);
    RMContext mockContext = mock(RMContext.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    dtr.setRMContext(mockContext);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
    dtr.init(conf);
    dtr.start();
    // submit a job that blocks during renewal                                 
    Thread submitThread = new Thread() {

        @Override
        public void run() {
            dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user", new Configuration());
        }
    };
    submitThread.start();
    // wait till 1st submit blocks, then submit another
    startBarrier.await();
    dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user", new Configuration());
    // signal 1st to complete                                                  
    endBarrier.await();
    submitThread.join();
}

Example 60 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestDelegationTokenRenewer method testDTKeepAlive1.

/**
   * Basic idea of the test:
   * 0. Setup token KEEP_ALIVE
   * 1. create tokens.
   * 2. register them for renewal - to be cancelled on app complete
   * 3. Complete app.
   * 4. Verify token is alive within the KEEP_ALIVE time
   * 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
   * @throws IOException
   * @throws URISyntaxException
   */
@Test(timeout = 60000)
public void testDTKeepAlive1() throws Exception {
    Configuration lconf = new Configuration(conf);
    lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    //Keep tokens alive for 6 seconds.
    lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
    //Try removing tokens every second.
    lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
    DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter);
    RMContext mockContext = mock(RMContext.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    localDtr.setRMContext(mockContext);
    localDtr.init(lconf);
    localDtr.start();
    MyFS dfs = (MyFS) FileSystem.get(lconf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());
    Credentials ts = new Credentials();
    // get the delegation tokens
    MyToken token1 = dfs.getDelegationToken("user1");
    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);
    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    localDtr.addApplicationAsync(applicationId_0, ts, true, "user", new Configuration());
    waitForEventsToGetProcessed(localDtr);
    if (!eventQueue.isEmpty()) {
        Event evt = eventQueue.take();
        if (evt instanceof RMAppEvent) {
            Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START);
        } else {
            fail("RMAppEvent.START was expected!!");
        }
    }
    localDtr.applicationFinished(applicationId_0);
    waitForEventsToGetProcessed(localDtr);
    //Token should still be around. Renewal should not fail.
    token1.renew(lconf);
    //Allow the keepalive time to run out
    Thread.sleep(10000l);
    //The token should have been cancelled at this point. Renewal will fail.
    try {
        token1.renew(lconf);
        fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
    }
}