Examples with TimelineClient org.apache.hadoop.yarn.client.api.TimelineClient used on opensource projects

Example 6 with TimelineClient

use of org.apache.hadoop.yarn.client.api.TimelineClient in project jstorm by alibaba.

the class JstormOnYarn method prepareTimelineDomain.

private void prepareTimelineDomain() {
    TimelineClient timelineClient = null;
    if (jstormClientContext.conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
        timelineClient = TimelineClient.createTimelineClient();
        timelineClient.init(jstormClientContext.conf);
        timelineClient.start();
    } else {
        LOG.warn("Cannot put the domain " + jstormClientContext.domainId + " because the timeline service is not enabled");
        return;
    }
    try {
        TimelineDomain domain = new TimelineDomain();
        domain.setId(jstormClientContext.domainId);
        domain.setReaders(jstormClientContext.viewACLs != null && jstormClientContext.viewACLs.length() > 0 ? jstormClientContext.viewACLs : JOYConstants.BLANK);
        domain.setWriters(jstormClientContext.modifyACLs != null && jstormClientContext.modifyACLs.length() > 0 ? jstormClientContext.modifyACLs : JOYConstants.BLANK);
        timelineClient.putDomain(domain);
        LOG.info("Put the timeline domain: " + TimelineUtils.dumpTimelineRecordtoJSON(domain));
    } catch (Exception e) {
        LOG.error("Error when putting the timeline domain", e);
    } finally {
        timelineClient.stop();
    }
}

Example 7 with TimelineClient

use of org.apache.hadoop.yarn.client.api.TimelineClient in project hadoop by apache.

the class SimpleEntityWriterV1 method map.

public void map(IntWritable key, IntWritable val, Context context) throws IOException {
    TimelineClient tlc = new TimelineClientImpl();
    Configuration conf = context.getConfiguration();
    final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);
    long totalTime = 0;
    final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
    final Random rand = new Random();
    final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
    final char[] payLoad = new char[kbs * 1024];
    for (int i = 0; i < testtimes; i++) {
        // Generate a fixed length random payload
        for (int xx = 0; xx < kbs * 1024; xx++) {
            int alphaNumIdx = rand.nextInt(ALPHA_NUMS.length);
            payLoad[xx] = ALPHA_NUMS[alphaNumIdx];
        }
        String entId = taskAttemptId + "_" + Integer.toString(i);
        final TimelineEntity entity = new TimelineEntity();
        entity.setEntityId(entId);
        entity.setEntityType("FOO_ATTEMPT");
        entity.addOtherInfo("PERF_TEST", payLoad);
        // add an event
        TimelineEvent event = new TimelineEvent();
        event.setTimestamp(System.currentTimeMillis());
        event.setEventType("foo_event");
        entity.addEvent(event);
        // use the current user for this purpose
        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
        long startWrite = System.nanoTime();
        try {
            tlc.putEntities(entity);
        } catch (Exception e) {
            context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).increment(1);
            LOG.error("writing to the timeline service failed", e);
        }
        long endWrite = System.nanoTime();
        totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite - startWrite);
    }
    LOG.info("wrote " + testtimes + " entities (" + kbs * testtimes + " kB) in " + totalTime + " ms");
    context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).increment(totalTime);
    context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).increment(testtimes);
    context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).increment(kbs * testtimes);
}

Example 8 with TimelineClient

use of org.apache.hadoop.yarn.client.api.TimelineClient in project hadoop by apache.

the class TestTimelineAuthenticationFilter method createTimelineClientForUGI.

private TimelineClient createTimelineClientForUGI() {
    TimelineClient client = TimelineClient.createTimelineClient();
    client.init(conf);
    client.start();
    return client;
}

Example 9 with TimelineClient

use of org.apache.hadoop.yarn.client.api.TimelineClient in project hadoop by apache.

the class TestTimelineAuthenticationFilter method testDelegationTokenOperations.

@Test
public void testDelegationTokenOperations() throws Exception {
    TimelineClient httpUserClient = KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<TimelineClient>() {

        @Override
        public TimelineClient call() throws Exception {
            return createTimelineClientForUGI();
        }
    });
    UserGroupInformation httpUser = KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<UserGroupInformation>() {

        @Override
        public UserGroupInformation call() throws Exception {
            return UserGroupInformation.getCurrentUser();
        }
    });
    // Let HTTP user to get the delegation for itself
    Token<TimelineDelegationTokenIdentifier> token = httpUserClient.getDelegationToken(httpUser.getShortUserName());
    Assert.assertNotNull(token);
    TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier();
    Assert.assertNotNull(tDT);
    Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner());
    // Renew token
    Assert.assertFalse(token.getService().toString().isEmpty());
    // Renew the token from the token service address
    long renewTime1 = httpUserClient.renewDelegationToken(token);
    Thread.sleep(100);
    token.setService(new Text());
    Assert.assertTrue(token.getService().toString().isEmpty());
    // If the token service address is not avaiable, it still can be renewed
    // from the configured address
    long renewTime2 = httpUserClient.renewDelegationToken(token);
    Assert.assertTrue(renewTime1 < renewTime2);
    // Cancel token
    Assert.assertTrue(token.getService().toString().isEmpty());
    // If the token service address is not avaiable, it still can be canceled
    // from the configured address
    httpUserClient.cancelDelegationToken(token);
    // Renew should not be successful because the token is canceled
    try {
        httpUserClient.renewDelegationToken(token);
        Assert.fail();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Renewal request for unknown token"));
    }
    // Let HTTP user to get the delegation token for FOO user
    UserGroupInformation fooUgi = UserGroupInformation.createProxyUser(FOO_USER, httpUser);
    TimelineClient fooUserClient = fooUgi.doAs(new PrivilegedExceptionAction<TimelineClient>() {

        @Override
        public TimelineClient run() throws Exception {
            return createTimelineClientForUGI();
        }
    });
    token = fooUserClient.getDelegationToken(httpUser.getShortUserName());
    Assert.assertNotNull(token);
    tDT = token.decodeIdentifier();
    Assert.assertNotNull(tDT);
    Assert.assertEquals(new Text(FOO_USER), tDT.getOwner());
    Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser());
    // Renew token as the renewer
    final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token;
    renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew);
    renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew);
    Assert.assertTrue(renewTime1 < renewTime2);
    // Cancel token
    Assert.assertFalse(tokenToRenew.getService().toString().isEmpty());
    // Cancel the token from the token service address
    fooUserClient.cancelDelegationToken(tokenToRenew);
    // Renew should not be successful because the token is canceled
    try {
        httpUserClient.renewDelegationToken(tokenToRenew);
        Assert.fail();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Renewal request for unknown token"));
    }
    // Let HTTP user to get the delegation token for BAR user
    UserGroupInformation barUgi = UserGroupInformation.createProxyUser(BAR_USER, httpUser);
    TimelineClient barUserClient = barUgi.doAs(new PrivilegedExceptionAction<TimelineClient>() {

        @Override
        public TimelineClient run() {
            return createTimelineClientForUGI();
        }
    });
    try {
        barUserClient.getDelegationToken(httpUser.getShortUserName());
        Assert.fail();
    } catch (Exception e) {
        Assert.assertTrue(e.getCause() instanceof AuthorizationException || e.getCause() instanceof AuthenticationException);
    }
}