Examples with DataInputByteBuffer org.apache.hadoop.io.DataInputByteBuffer used on opensource projects

Example 1 with DataInputByteBuffer

use of org.apache.hadoop.io.DataInputByteBuffer in project hadoop by apache.

the class AMLauncher method setupTokens.

@Private
@VisibleForTesting
protected void setupTokens(ContainerLaunchContext container, ContainerId containerID) throws IOException {
    Map<String, String> environment = container.getEnvironment();
    environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase());
    // Set AppSubmitTime to be consumable by the AM.
    ApplicationId applicationId = application.getAppAttemptId().getApplicationId();
    environment.put(ApplicationConstants.APP_SUBMIT_TIME_ENV, String.valueOf(rmContext.getRMApps().get(applicationId).getSubmitTime()));
    Credentials credentials = new Credentials();
    DataInputByteBuffer dibb = new DataInputByteBuffer();
    ByteBuffer tokens = container.getTokens();
    if (tokens != null) {
        // TODO: Don't do this kind of checks everywhere.
        dibb.reset(tokens);
        credentials.readTokenStorageStream(dibb);
        tokens.rewind();
    }
    // Add AMRMToken
    Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken();
    if (amrmToken != null) {
        credentials.addToken(amrmToken.getService(), amrmToken);
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}

Example 2 with DataInputByteBuffer

use of org.apache.hadoop.io.DataInputByteBuffer in project hadoop by apache.

the class ShuffleHandler method deserializeServiceData.

static Token<JobTokenIdentifier> deserializeServiceData(ByteBuffer secret) throws IOException {
    DataInputByteBuffer in = new DataInputByteBuffer();
    in.reset(secret);
    Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>();
    jt.readFields(in);
    return jt;
}

Example 3 with DataInputByteBuffer

use of org.apache.hadoop.io.DataInputByteBuffer in project hadoop by apache.

the class NodeStatusUpdaterImpl method parseCredentials.

private static Map<ApplicationId, Credentials> parseCredentials(Map<ApplicationId, ByteBuffer> systemCredentials) throws IOException {
    Map<ApplicationId, Credentials> map = new HashMap<ApplicationId, Credentials>();
    for (Map.Entry<ApplicationId, ByteBuffer> entry : systemCredentials.entrySet()) {
        Credentials credentials = new Credentials();
        DataInputByteBuffer buf = new DataInputByteBuffer();
        ByteBuffer buffer = entry.getValue();
        buffer.rewind();
        buf.reset(buffer);
        credentials.readTokenStorageStream(buf);
        map.put(entry.getKey(), credentials);
    }
    if (LOG.isDebugEnabled()) {
        for (Map.Entry<ApplicationId, Credentials> entry : map.entrySet()) {
            LOG.debug("Retrieved credentials form RM for " + entry.getKey() + ": " + entry.getValue().getAllTokens());
        }
    }
    return map;
}

Example 4 with DataInputByteBuffer

use of org.apache.hadoop.io.DataInputByteBuffer in project hadoop by apache.

the class TestDelegationTokenRenewer method testRMRestartWithExpiredToken.

// 1. token is expired before app completes.
// 2. RM shutdown.
// 3. When RM recovers the app, token renewal will fail as token expired.
//    RM should request a new token and sent it to NM for log-aggregation.
@Test
public void testRMRestartWithExpiredToken() throws Exception {
    Configuration yarnConf = new YarnConfiguration();
    yarnConf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
    yarnConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    yarnConf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
    yarnConf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    UserGroupInformation.setConfiguration(yarnConf);
    // create Token1:
    Text userText1 = new Text("user1");
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<DelegationTokenIdentifier> originalToken = new Token<>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(userText1, originalToken);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(yarnConf);
    MockRM rm1 = new TestSecurityMockRM(yarnConf, memStore);
    rm1.start();
    RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, credentials);
    // create token2
    Text userText2 = new Text("user1");
    DelegationTokenIdentifier dtId2 = new DelegationTokenIdentifier(userText1, new Text("renewer2"), userText2);
    final Token<DelegationTokenIdentifier> updatedToken = new Token<DelegationTokenIdentifier>(dtId2.getBytes(), "password2".getBytes(), dtId2.getKind(), new Text("service2"));
    AtomicBoolean firstRenewInvoked = new AtomicBoolean(false);
    AtomicBoolean secondRenewInvoked = new AtomicBoolean(false);
    MockRM rm2 = new TestSecurityMockRM(yarnConf, memStore) {

        @Override
        protected DelegationTokenRenewer createDelegationTokenRenewer() {
            return new DelegationTokenRenewer() {

                @Override
                protected void renewToken(final DelegationTokenToRenew dttr) throws IOException {
                    if (dttr.token.equals(updatedToken)) {
                        secondRenewInvoked.set(true);
                        super.renewToken(dttr);
                    } else if (dttr.token.equals(originalToken)) {
                        firstRenewInvoked.set(true);
                        throw new InvalidToken("Failed to renew");
                    } else {
                        throw new IOException("Unexpected");
                    }
                }

                @Override
                protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException {
                    credentials.addToken(updatedToken.getService(), updatedToken);
                    return new Token<?>[] { updatedToken };
                }
            };
        }
    };
    // simulating restart the rm
    rm2.start();
    // check nm can retrieve the token
    final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());
    nm1.registerNode();
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    ByteBuffer tokenBuffer = response.getSystemCredentialsForApps().get(app.getApplicationId());
    Assert.assertNotNull(tokenBuffer);
    Credentials appCredentials = new Credentials();
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokenBuffer.rewind();
    buf.reset(tokenBuffer);
    appCredentials.readTokenStorageStream(buf);
    Assert.assertTrue(firstRenewInvoked.get() && secondRenewInvoked.get());
    Assert.assertTrue(appCredentials.getAllTokens().contains(updatedToken));
}

Example 5 with DataInputByteBuffer

use of org.apache.hadoop.io.DataInputByteBuffer in project hadoop by apache.

the class TestDelegationTokenRenewer method testAppSubmissionWithoutDelegationToken.

// YARN will get the token for the app submitted without the delegation token.
@Test
public void testAppSubmissionWithoutDelegationToken() throws Exception {
    conf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
    // create token2
    Text userText2 = new Text("user2");
    DelegationTokenIdentifier dtId2 = new DelegationTokenIdentifier(new Text("user2"), new Text("renewer2"), userText2);
    final Token<DelegationTokenIdentifier> token2 = new Token<DelegationTokenIdentifier>(dtId2.getBytes(), "password2".getBytes(), dtId2.getKind(), new Text("service2"));
    final MockRM rm = new TestSecurityMockRM(conf, null) {

        @Override
        protected DelegationTokenRenewer createDelegationTokenRenewer() {
            return new DelegationTokenRenewer() {

                @Override
                protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException {
                    credentials.addToken(token2.getService(), token2);
                    return new Token<?>[] { token2 };
                }
            };
        }
    };
    rm.start();
    // submit an app without delegationToken
    RMApp app = rm.submitApp(200);
    // wait for the new retrieved hdfs token.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            return rm.getRMContext().getDelegationTokenRenewer().getDelegationTokens().contains(token2);
        }
    }, 1000, 20000);
    // check nm can retrieve the token
    final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm1.registerNode();
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    ByteBuffer tokenBuffer = response.getSystemCredentialsForApps().get(app.getApplicationId());
    Assert.assertNotNull(tokenBuffer);
    Credentials appCredentials = new Credentials();
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokenBuffer.rewind();
    buf.reset(tokenBuffer);
    appCredentials.readTokenStorageStream(buf);
    Assert.assertTrue(appCredentials.getAllTokens().contains(token2));
}