Examples with TimelineDelegationTokenIdentifier org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier used on opensource projects

Example 1 with TimelineDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier in project hadoop by apache.

the class TestYarnClient method testAutomaticTimelineDelegationTokenLoading.

@Test
public void testAutomaticTimelineDelegationTokenLoading() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
    TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier();
    final Token<TimelineDelegationTokenIdentifier> dToken = new Token<TimelineDelegationTokenIdentifier>(timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
    // create a mock client
    YarnClientImpl client = spy(new YarnClientImpl() {

        @Override
        TimelineClient createTimelineClient() throws IOException, YarnException {
            timelineClient = mock(TimelineClient.class);
            when(timelineClient.getDelegationToken(any(String.class))).thenReturn(dToken);
            return timelineClient;
        }

        @Override
        protected void serviceStart() throws Exception {
            rmClient = mock(ApplicationClientProtocol.class);
        }

        @Override
        protected void serviceStop() throws Exception {
        }

        @Override
        public ApplicationReport getApplicationReport(ApplicationId appId) {
            ApplicationReport report = mock(ApplicationReport.class);
            when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING);
            return report;
        }

        @Override
        public boolean isSecurityEnabled() {
            return true;
        }
    });
    client.init(conf);
    client.start();
    try {
        // when i == 1, timeline DT doesn't exist, need to get one more
        for (int i = 0; i < 2; ++i) {
            ApplicationSubmissionContext context = mock(ApplicationSubmissionContext.class);
            ApplicationId applicationId = ApplicationId.newInstance(0, i + 1);
            when(context.getApplicationId()).thenReturn(applicationId);
            DataOutputBuffer dob = new DataOutputBuffer();
            Credentials credentials = new Credentials();
            if (i == 0) {
                credentials.addToken(client.timelineService, dToken);
            }
            credentials.writeTokenStorageToStream(dob);
            ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
            ContainerLaunchContext clc = ContainerLaunchContext.newInstance(null, null, null, null, tokens, null);
            when(context.getAMContainerSpec()).thenReturn(clc);
            client.submitApplication(context);
            if (i == 0) {
                // GetTimelineDelegationToken shouldn't be called
                verify(client, never()).getTimelineDelegationToken();
            }
            // In either way, token should be there
            credentials = new Credentials();
            DataInputByteBuffer dibb = new DataInputByteBuffer();
            tokens = clc.getTokens();
            if (tokens != null) {
                dibb.reset(tokens);
                credentials.readTokenStorageStream(dibb);
                tokens.rewind();
            }
            Collection<Token<? extends TokenIdentifier>> dTokens = credentials.getAllTokens();
            Assert.assertEquals(1, dTokens.size());
            Assert.assertEquals(dToken, dTokens.iterator().next());
        }
    } finally {
        client.stop();
    }
}

Example 2 with TimelineDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier in project hadoop by apache.

the class TimelineClientImpl method putTimelineDataInJSONFile.

/**
   * Put timeline data in a JSON file via command line.
   * 
   * @param path
   *          path to the timeline data JSON file
   * @param type
   *          the type of the timeline data in the JSON file
   */
private static void putTimelineDataInJSONFile(String path, String type) {
    File jsonFile = new File(path);
    if (!jsonFile.exists()) {
        LOG.error("File [" + jsonFile.getAbsolutePath() + "] doesn't exist");
        return;
    }
    YarnJacksonJaxbJsonProvider.configObjectMapper(MAPPER);
    TimelineEntities entities = null;
    TimelineDomains domains = null;
    try {
        if (type.equals(ENTITY_DATA_TYPE)) {
            entities = MAPPER.readValue(jsonFile, TimelineEntities.class);
        } else if (type.equals(DOMAIN_DATA_TYPE)) {
            domains = MAPPER.readValue(jsonFile, TimelineDomains.class);
        }
    } catch (Exception e) {
        LOG.error("Error when reading  " + e.getMessage());
        e.printStackTrace(System.err);
        return;
    }
    Configuration conf = new YarnConfiguration();
    TimelineClient client = TimelineClient.createTimelineClient();
    client.init(conf);
    client.start();
    try {
        if (UserGroupInformation.isSecurityEnabled() && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
            Token<TimelineDelegationTokenIdentifier> token = client.getDelegationToken(UserGroupInformation.getCurrentUser().getUserName());
            UserGroupInformation.getCurrentUser().addToken(token);
        }
        if (type.equals(ENTITY_DATA_TYPE)) {
            TimelinePutResponse response = client.putEntities(entities.getEntities().toArray(new TimelineEntity[entities.getEntities().size()]));
            if (response.getErrors().size() == 0) {
                LOG.info("Timeline entities are successfully put");
            } else {
                for (TimelinePutResponse.TimelinePutError error : response.getErrors()) {
                    LOG.error("TimelineEntity [" + error.getEntityType() + ":" + error.getEntityId() + "] is not successfully put. Error code: " + error.getErrorCode());
                }
            }
        } else if (type.equals(DOMAIN_DATA_TYPE) && domains != null) {
            boolean hasError = false;
            for (TimelineDomain domain : domains.getDomains()) {
                try {
                    client.putDomain(domain);
                } catch (Exception e) {
                    LOG.error("Error when putting domain " + domain.getId(), e);
                    hasError = true;
                }
            }
            if (!hasError) {
                LOG.info("Timeline domains are successfully put");
            }
        }
    } catch (RuntimeException e) {
        LOG.error("Error when putting the timeline data", e);
    } catch (Exception e) {
        LOG.error("Error when putting the timeline data", e);
    } finally {
        client.stop();
    }
}

Example 3 with TimelineDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier in project hadoop by apache.

the class TestYARNTokenIdentifier method testTimelineDelegationTokenIdentifier.

@Test
public void testTimelineDelegationTokenIdentifier() throws IOException {
    Text owner = new Text("user1");
    Text renewer = new Text("user2");
    Text realUser = new Text("user3");
    long issueDate = 1;
    long maxDate = 2;
    int sequenceNumber = 3;
    int masterKeyId = 4;
    TimelineDelegationTokenIdentifier token = new TimelineDelegationTokenIdentifier(owner, renewer, realUser);
    token.setIssueDate(issueDate);
    token.setMaxDate(maxDate);
    token.setSequenceNumber(sequenceNumber);
    token.setMasterKeyId(masterKeyId);
    TimelineDelegationTokenIdentifier anotherToken = new TimelineDelegationTokenIdentifier();
    byte[] tokenContent = token.getBytes();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenContent, tokenContent.length);
    anotherToken.readFields(dib);
    // verify the whole record equals with original record
    Assert.assertEquals("Token is not the same after serialization " + "and deserialization.", token, anotherToken);
    Assert.assertEquals("owner from proto is not the same with original token", anotherToken.getOwner(), owner);
    Assert.assertEquals("renewer from proto is not the same with original token", anotherToken.getRenewer(), renewer);
    Assert.assertEquals("realUser from proto is not the same with original token", anotherToken.getRealUser(), realUser);
    Assert.assertEquals("issueDate from proto is not the same with original token", anotherToken.getIssueDate(), issueDate);
    Assert.assertEquals("maxDate from proto is not the same with original token", anotherToken.getMaxDate(), maxDate);
    Assert.assertEquals("sequenceNumber from proto is not the same with original token", anotherToken.getSequenceNumber(), sequenceNumber);
    Assert.assertEquals("masterKeyId from proto is not the same with original token", anotherToken.getMasterKeyId(), masterKeyId);
}

Example 4 with TimelineDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier in project hadoop by apache.

the class TestYARNTokenIdentifier method testParseTimelineDelegationTokenIdentifierRenewer.

@Test
public void testParseTimelineDelegationTokenIdentifierRenewer() throws IOException {
    // Server side when generation a timeline DT
    Configuration conf = new YarnConfiguration();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[2:$1@$0]([nr]m@.*EXAMPLE.COM)s/.*/yarn/");
    HadoopKerberosName.setConfiguration(conf);
    Text owner = new Text("owner");
    Text renewer = new Text("rm/localhost@EXAMPLE.COM");
    Text realUser = new Text("realUser");
    TimelineDelegationTokenIdentifier token = new TimelineDelegationTokenIdentifier(owner, renewer, realUser);
    Assert.assertEquals(new Text("yarn"), token.getRenewer());
}

Example 5 with TimelineDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier in project hadoop by apache.

the class TestTimelineClient method testDelegationTokenOperationsRetry.

@Test
public void testDelegationTokenOperationsRetry() throws Exception {
    int newMaxRetries = 5;
    long newIntervalMs = 500;
    YarnConfiguration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, newMaxRetries);
    conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, newIntervalMs);
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    // use kerberos to bypass the issue in HADOOP-11215
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    TimelineClientImpl client = createTimelineClient(conf);
    TimelineClientImpl clientFake = createTimelineClientFakeTimelineClientRetryOp(conf);
    TestTimlineDelegationTokenSecretManager dtManager = new TestTimlineDelegationTokenSecretManager();
    try {
        dtManager.startThreads();
        Thread.sleep(3000);
        try {
            // try getting a delegation token
            client.getDelegationToken(UserGroupInformation.getCurrentUser().getShortUserName());
            assertFail();
        } catch (RuntimeException ce) {
            assertException(client, ce);
        }
        try {
            // try renew a delegation token
            TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier(new Text("tester"), new Text("tester"), new Text("tester"));
            client.renewDelegationToken(new Token<TimelineDelegationTokenIdentifier>(timelineDT.getBytes(), dtManager.createPassword(timelineDT), timelineDT.getKind(), new Text("0.0.0.0:8188")));
            assertFail();
        } catch (RuntimeException ce) {
            assertException(client, ce);
        }
        try {
            // try cancel a delegation token
            TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier(new Text("tester"), new Text("tester"), new Text("tester"));
            client.cancelDelegationToken(new Token<TimelineDelegationTokenIdentifier>(timelineDT.getBytes(), dtManager.createPassword(timelineDT), timelineDT.getKind(), new Text("0.0.0.0:8188")));
            assertFail();
        } catch (RuntimeException ce) {
            assertException(client, ce);
        }
        // Test DelegationTokenOperationsRetry on SocketTimeoutException
        try {
            TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier(new Text("tester"), new Text("tester"), new Text("tester"));
            clientFake.cancelDelegationToken(new Token<TimelineDelegationTokenIdentifier>(timelineDT.getBytes(), dtManager.createPassword(timelineDT), timelineDT.getKind(), new Text("0.0.0.0:8188")));
            assertFail();
        } catch (RuntimeException ce) {
            assertException(clientFake, ce);
        }
    } finally {
        client.stop();
        clientFake.stop();
        dtManager.stopThreads();
    }
}