Examples with HistoryServerStateStoreService org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService used on opensource projects

Example 1 with HistoryServerStateStoreService

use of org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService in project hadoop by apache.

the class TestHistoryServerFileSystemStateStoreService method testUpdatedTokenRecovery.

@Test
public void testUpdatedTokenRecovery() throws IOException {
    IOException intentionalErr = new IOException("intentional error");
    FileSystem fs = FileSystem.getLocal(conf);
    final FileSystem spyfs = spy(fs);
    // make the update token process fail halfway through where we're left
    // with just the temporary update file and no token file
    ArgumentMatcher<Path> updateTmpMatcher = new ArgumentMatcher<Path>() {

        @Override
        public boolean matches(Object argument) {
            if (argument instanceof Path) {
                return ((Path) argument).getName().startsWith("update");
            }
            return false;
        }
    };
    doThrow(intentionalErr).when(spyfs).rename(argThat(updateTmpMatcher), isA(Path.class));
    conf.set(JHAdminConfig.MR_HS_FS_STATE_STORE_URI, testDir.getAbsoluteFile().toURI().toString());
    HistoryServerStateStoreService store = new HistoryServerFileSystemStateStoreService() {

        @Override
        FileSystem createFileSystem() throws IOException {
            return spyfs;
        }
    };
    store.init(conf);
    store.start();
    final MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"), new Text("tokenRenewer1"), new Text("tokenUser1"));
    token1.setSequenceNumber(1);
    final Long tokenDate1 = 1L;
    store.storeToken(token1, tokenDate1);
    final Long newTokenDate1 = 975318642L;
    try {
        store.updateToken(token1, newTokenDate1);
        fail("intentional error not thrown");
    } catch (IOException e) {
        assertEquals(intentionalErr, e);
    }
    store.close();
    // verify the update file is seen and parsed upon recovery when
    // original token file is missing
    store = createAndStartStore();
    HistoryServerState state = store.loadState();
    assertEquals("incorrect loaded token count", 1, state.tokenState.size());
    assertTrue("missing token 1", state.tokenState.containsKey(token1));
    assertEquals("incorrect token 1 date", newTokenDate1, state.tokenState.get(token1));
    store.close();
}

Example 2 with HistoryServerStateStoreService

use of org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService in project hadoop by apache.

the class TestHistoryServerFileSystemStateStoreService method testTokenStore.

private void testTokenStore(String stateStoreUri) throws IOException {
    conf.set(JHAdminConfig.MR_HS_FS_STATE_STORE_URI, stateStoreUri);
    HistoryServerStateStoreService store = createAndStartStore();
    HistoryServerState state = store.loadState();
    assertTrue("token state not empty", state.tokenState.isEmpty());
    assertTrue("key state not empty", state.tokenMasterKeyState.isEmpty());
    final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes());
    final MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"), new Text("tokenRenewer1"), new Text("tokenUser1"));
    token1.setSequenceNumber(1);
    final Long tokenDate1 = 1L;
    final MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"), new Text("tokenRenewer2"), new Text("tokenUser2"));
    token2.setSequenceNumber(12345678);
    final Long tokenDate2 = 87654321L;
    store.storeTokenMasterKey(key1);
    try {
        store.storeTokenMasterKey(key1);
        fail("redundant store of key undetected");
    } catch (IOException e) {
    // expected
    }
    store.storeToken(token1, tokenDate1);
    store.storeToken(token2, tokenDate2);
    try {
        store.storeToken(token1, tokenDate1);
        fail("redundant store of token undetected");
    } catch (IOException e) {
    // expected
    }
    store.close();
    store = createAndStartStore();
    state = store.loadState();
    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
    assertTrue("missing token 1", state.tokenState.containsKey(token1));
    assertEquals("incorrect token 1 date", tokenDate1, state.tokenState.get(token1));
    assertTrue("missing token 2", state.tokenState.containsKey(token2));
    assertEquals("incorrect token 2 date", tokenDate2, state.tokenState.get(token2));
    assertEquals("incorrect master key count", 1, state.tokenMasterKeyState.size());
    assertTrue("missing master key 1", state.tokenMasterKeyState.contains(key1));
    final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes());
    final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes());
    final MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"), new Text("tokenRenewer3"), new Text("tokenUser3"));
    token3.setSequenceNumber(12345679);
    final Long tokenDate3 = 87654321L;
    store.removeToken(token1);
    store.storeTokenMasterKey(key2);
    final Long newTokenDate2 = 975318642L;
    store.updateToken(token2, newTokenDate2);
    store.removeTokenMasterKey(key1);
    store.storeTokenMasterKey(key3);
    store.storeToken(token3, tokenDate3);
    store.close();
    store = createAndStartStore();
    state = store.loadState();
    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
    assertFalse("token 1 not removed", state.tokenState.containsKey(token1));
    assertTrue("missing token 2", state.tokenState.containsKey(token2));
    assertEquals("incorrect token 2 date", newTokenDate2, state.tokenState.get(token2));
    assertTrue("missing token 3", state.tokenState.containsKey(token3));
    assertEquals("incorrect token 3 date", tokenDate3, state.tokenState.get(token3));
    assertEquals("incorrect master key count", 2, state.tokenMasterKeyState.size());
    assertFalse("master key 1 not removed", state.tokenMasterKeyState.contains(key1));
    assertTrue("missing master key 2", state.tokenMasterKeyState.contains(key2));
    assertTrue("missing master key 3", state.tokenMasterKeyState.contains(key3));
}

Example 3 with HistoryServerStateStoreService

use of org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService in project hadoop by apache.

the class TestHistoryServerLeveldbStateStoreService method testTokenStore.

@Test
public void testTokenStore() throws IOException {
    HistoryServerStateStoreService store = createAndStartStore();
    // verify initially the store is empty
    HistoryServerState state = store.loadState();
    assertTrue("token state not empty", state.tokenState.isEmpty());
    assertTrue("key state not empty", state.tokenMasterKeyState.isEmpty());
    // store a key and some tokens
    final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes());
    final MRDelegationTokenIdentifier token1 = new MRDelegationTokenIdentifier(new Text("tokenOwner1"), new Text("tokenRenewer1"), new Text("tokenUser1"));
    token1.setSequenceNumber(1);
    final Long tokenDate1 = 1L;
    final MRDelegationTokenIdentifier token2 = new MRDelegationTokenIdentifier(new Text("tokenOwner2"), new Text("tokenRenewer2"), new Text("tokenUser2"));
    token2.setSequenceNumber(12345678);
    final Long tokenDate2 = 87654321L;
    store.storeTokenMasterKey(key1);
    store.storeToken(token1, tokenDate1);
    store.storeToken(token2, tokenDate2);
    store.close();
    // verify the key and tokens can be recovered
    store = createAndStartStore();
    state = store.loadState();
    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
    assertTrue("missing token 1", state.tokenState.containsKey(token1));
    assertEquals("incorrect token 1 date", tokenDate1, state.tokenState.get(token1));
    assertTrue("missing token 2", state.tokenState.containsKey(token2));
    assertEquals("incorrect token 2 date", tokenDate2, state.tokenState.get(token2));
    assertEquals("incorrect master key count", 1, state.tokenMasterKeyState.size());
    assertTrue("missing master key 1", state.tokenMasterKeyState.contains(key1));
    // store some more keys and tokens, remove the previous key and one
    // of the tokens, and renew a previous token
    final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes());
    final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes());
    final MRDelegationTokenIdentifier token3 = new MRDelegationTokenIdentifier(new Text("tokenOwner3"), new Text("tokenRenewer3"), new Text("tokenUser3"));
    token3.setSequenceNumber(12345679);
    final Long tokenDate3 = 87654321L;
    store.removeToken(token1);
    store.storeTokenMasterKey(key2);
    final Long newTokenDate2 = 975318642L;
    store.updateToken(token2, newTokenDate2);
    store.removeTokenMasterKey(key1);
    store.storeTokenMasterKey(key3);
    store.storeToken(token3, tokenDate3);
    store.close();
    // verify the new keys and tokens are recovered, the removed key and
    // token are no longer present, and the renewed token has the updated
    // expiration date
    store = createAndStartStore();
    state = store.loadState();
    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
    assertFalse("token 1 not removed", state.tokenState.containsKey(token1));
    assertTrue("missing token 2", state.tokenState.containsKey(token2));
    assertEquals("incorrect token 2 date", newTokenDate2, state.tokenState.get(token2));
    assertTrue("missing token 3", state.tokenState.containsKey(token3));
    assertEquals("incorrect token 3 date", tokenDate3, state.tokenState.get(token3));
    assertEquals("incorrect master key count", 2, state.tokenMasterKeyState.size());
    assertFalse("master key 1 not removed", state.tokenMasterKeyState.contains(key1));
    assertTrue("missing master key 2", state.tokenMasterKeyState.contains(key2));
    assertTrue("missing master key 3", state.tokenMasterKeyState.contains(key3));
    store.close();
}

Example 4 with HistoryServerStateStoreService

use of org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService in project hadoop by apache.

the class TestJHSDelegationTokenSecretManager method testRecovery.

@Test
public void testRecovery() throws IOException {
    Configuration conf = new Configuration();
    HistoryServerStateStoreService store = new HistoryServerMemStateStoreService();
    store.init(conf);
    store.start();
    JHSDelegationTokenSecretManagerForTest mgr = new JHSDelegationTokenSecretManagerForTest(store);
    mgr.startThreads();
    MRDelegationTokenIdentifier tokenId1 = new MRDelegationTokenIdentifier(new Text("tokenOwner"), new Text("tokenRenewer"), new Text("tokenUser"));
    Token<MRDelegationTokenIdentifier> token1 = new Token<MRDelegationTokenIdentifier>(tokenId1, mgr);
    MRDelegationTokenIdentifier tokenId2 = new MRDelegationTokenIdentifier(new Text("tokenOwner"), new Text("tokenRenewer"), new Text("tokenUser"));
    Token<MRDelegationTokenIdentifier> token2 = new Token<MRDelegationTokenIdentifier>(tokenId2, mgr);
    DelegationKey[] keys = mgr.getAllKeys();
    long tokenRenewDate1 = mgr.getAllTokens().get(tokenId1).getRenewDate();
    long tokenRenewDate2 = mgr.getAllTokens().get(tokenId2).getRenewDate();
    mgr.stopThreads();
    mgr = new JHSDelegationTokenSecretManagerForTest(store);
    mgr.recover(store.loadState());
    List<DelegationKey> recoveredKeys = Arrays.asList(mgr.getAllKeys());
    for (DelegationKey key : keys) {
        assertTrue("key missing after recovery", recoveredKeys.contains(key));
    }
    assertTrue("token1 missing", mgr.getAllTokens().containsKey(tokenId1));
    assertEquals("token1 renew date", tokenRenewDate1, mgr.getAllTokens().get(tokenId1).getRenewDate());
    assertTrue("token2 missing", mgr.getAllTokens().containsKey(tokenId2));
    assertEquals("token2 renew date", tokenRenewDate2, mgr.getAllTokens().get(tokenId2).getRenewDate());
    mgr.startThreads();
    mgr.verifyToken(tokenId1, token1.getPassword());
    mgr.verifyToken(tokenId2, token2.getPassword());
    MRDelegationTokenIdentifier tokenId3 = new MRDelegationTokenIdentifier(new Text("tokenOwner"), new Text("tokenRenewer"), new Text("tokenUser"));
    Token<MRDelegationTokenIdentifier> token3 = new Token<MRDelegationTokenIdentifier>(tokenId3, mgr);
    assertEquals("sequence number restore", tokenId2.getSequenceNumber() + 1, tokenId3.getSequenceNumber());
    mgr.cancelToken(token1, "tokenOwner");
    // Testing with full principal name
    MRDelegationTokenIdentifier tokenIdFull = new MRDelegationTokenIdentifier(new Text("tokenOwner/localhost@LOCALHOST"), new Text("tokenRenewer"), new Text("tokenUser"));
    KerberosName.setRules("RULE:[1:$1]\nRULE:[2:$1]");
    Token<MRDelegationTokenIdentifier> tokenFull = new Token<MRDelegationTokenIdentifier>(tokenIdFull, mgr);
    // Negative test
    try {
        mgr.cancelToken(tokenFull, "tokenOwner");
    } catch (AccessControlException ace) {
        assertTrue(ace.getMessage().contains("is not authorized to cancel the token"));
    }
    // Succeed to cancel with full principal
    mgr.cancelToken(tokenFull, tokenIdFull.getOwner().toString());
    long tokenRenewDate3 = mgr.getAllTokens().get(tokenId3).getRenewDate();
    mgr.stopThreads();
    mgr = new JHSDelegationTokenSecretManagerForTest(store);
    mgr.recover(store.loadState());
    assertFalse("token1 should be missing", mgr.getAllTokens().containsKey(tokenId1));
    assertTrue("token2 missing", mgr.getAllTokens().containsKey(tokenId2));
    assertEquals("token2 renew date", tokenRenewDate2, mgr.getAllTokens().get(tokenId2).getRenewDate());
    assertTrue("token3 missing", mgr.getAllTokens().containsKey(tokenId3));
    assertEquals("token3 renew date", tokenRenewDate3, mgr.getAllTokens().get(tokenId3).getRenewDate());
    mgr.startThreads();
    mgr.verifyToken(tokenId2, token2.getPassword());
    mgr.verifyToken(tokenId3, token3.getPassword());
    mgr.stopThreads();
}

Example 5 with HistoryServerStateStoreService

use of org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService in project hadoop by apache.

the class TestJHSSecurity method testDelegationToken.

@Test
public void testDelegationToken() throws IOException, InterruptedException {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    final YarnConfiguration conf = new YarnConfiguration(new JobConf());
    // Just a random principle
    conf.set(JHAdminConfig.MR_HISTORY_PRINCIPAL, "RandomOrc/localhost@apache.org");
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    final long initialInterval = 10000l;
    final long maxLifetime = 20000l;
    final long renewInterval = 10000l;
    JobHistoryServer jobHistoryServer = null;
    MRClientProtocol clientUsingDT = null;
    long tokenFetchTime;
    try {
        jobHistoryServer = new JobHistoryServer() {

            protected void doSecureLogin(Configuration conf) throws IOException {
            // no keytab based login
            }

            ;

            @Override
            protected JHSDelegationTokenSecretManager createJHSSecretManager(Configuration conf, HistoryServerStateStoreService store) {
                return new JHSDelegationTokenSecretManager(initialInterval, maxLifetime, renewInterval, 3600000, store);
            }

            @Override
            protected HistoryClientService createHistoryClientService() {
                return new HistoryClientService(historyContext, this.jhsDTSecretManager) {

                    @Override
                    protected void initializeWebApp(Configuration conf) {
                    // Don't need it, skip.;
                    }
                };
            }
        };
        //      final JobHistoryServer jobHistoryServer = jhServer;
        jobHistoryServer.init(conf);
        jobHistoryServer.start();
        final MRClientProtocol hsService = jobHistoryServer.getClientService().getClientHandler();
        // Fake the authentication-method
        UserGroupInformation loggedInUser = UserGroupInformation.createRemoteUser("testrenewer@APACHE.ORG");
        Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
        // Default realm is APACHE.ORG
        loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
        Token token = getDelegationToken(loggedInUser, hsService, loggedInUser.getShortUserName());
        tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to JHS using the delegation token
        clientUsingDT = getMRClientProtocol(token, jobHistoryServer.getClientService().getBindAddress(), "TheDarkLord", conf);
        GetJobReportRequest jobReportRequest = Records.newRecord(GetJobReportRequest.class);
        jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
        try {
            clientUsingDT.getJobReport(jobReportRequest);
        } catch (IOException e) {
            Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
        }
        // Renew after 50% of token age.
        while (System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
            Thread.sleep(500l);
        }
        long nextExpTime = renewDelegationToken(loggedInUser, hsService, token);
        long renewalTime = System.currentTimeMillis();
        LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
        // Wait for first expiry, but before renewed expiry.
        while (System.currentTimeMillis() > tokenFetchTime + initialInterval && System.currentTimeMillis() < nextExpTime) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        // Valid token because of renewal.
        try {
            clientUsingDT.getJobReport(jobReportRequest);
        } catch (IOException e) {
            Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
        }
        // Wait for expiry.
        while (System.currentTimeMillis() < renewalTime + renewInterval) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
        // Token should have expired.      
        try {
            clientUsingDT.getJobReport(jobReportRequest);
            fail("Should not have succeeded with an expired token");
        } catch (IOException e) {
            assertTrue(e.getCause().getMessage().contains("is expired"));
        }
        // Stop the existing proxy, start another.
        if (clientUsingDT != null) {
            //        RPC.stopProxy(clientUsingDT);
            clientUsingDT = null;
        }
        token = getDelegationToken(loggedInUser, hsService, loggedInUser.getShortUserName());
        tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to HSService using the delegation token
        clientUsingDT = getMRClientProtocol(token, jobHistoryServer.getClientService().getBindAddress(), "loginuser2", conf);
        try {
            clientUsingDT.getJobReport(jobReportRequest);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        }
        cancelDelegationToken(loggedInUser, hsService, token);
        // Testing the token with different renewer to cancel the token
        Token tokenWithDifferentRenewer = getDelegationToken(loggedInUser, hsService, "yarn");
        cancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer);
        if (clientUsingDT != null) {
            //        RPC.stopProxy(clientUsingDT);
            clientUsingDT = null;
        }
        // Creating a new connection.
        clientUsingDT = getMRClientProtocol(token, jobHistoryServer.getClientService().getBindAddress(), "loginuser2", conf);
        LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
        // Verify cancellation worked.
        try {
            clientUsingDT.getJobReport(jobReportRequest);
            fail("Should not have succeeded with a cancelled delegation token");
        } catch (IOException e) {
        }
    } finally {
        jobHistoryServer.stop();
    }
}