Examples with TimelineEntity org.apache.hadoop.yarn.api.records.timeline.TimelineEntity used on opensource projects

Example 51 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestSystemMetricsPublisher method testPublishApplicationMetrics.

@Test(timeout = 10000)
public void testPublishApplicationMetrics() throws Exception {
    long stateUpdateTimeStamp = System.currentTimeMillis();
    for (int i = 1; i <= 2; ++i) {
        ApplicationId appId = ApplicationId.newInstance(0, i);
        RMApp app = createRMApp(appId);
        metricsPublisher.appCreated(app, app.getStartTime());
        if (i == 1) {
            when(app.getQueue()).thenReturn("new test queue");
            ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
            when(asc.getUnmanagedAM()).thenReturn(false);
            when(asc.getPriority()).thenReturn(Priority.newInstance(1));
            when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
            ContainerLaunchContext containerLaunchContext = mock(ContainerLaunchContext.class);
            when(containerLaunchContext.getCommands()).thenReturn(Collections.singletonList("java -Xmx1024m"));
            when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
            when(app.getApplicationSubmissionContext()).thenReturn(asc);
            when(app.getApplicationPriority()).thenReturn(Priority.newInstance(1));
            metricsPublisher.appUpdated(app, 4L);
        } else {
            metricsPublisher.appUpdated(app, 4L);
        }
        metricsPublisher.appStateUpdated(app, YarnApplicationState.RUNNING, stateUpdateTimeStamp);
        metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
        if (i == 1) {
            metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
        } else {
            // in case user doesn't specify the ACLs
            metricsPublisher.appACLsUpdated(app, null, 4L);
        }
        TimelineEntity entity = null;
        do {
            entity = store.getEntity(appId.toString(), ApplicationMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class));
        // ensure Five events are both published before leaving the loop
        } while (entity == null || entity.getEvents().size() < 5);
        // verify all the fields
        Assert.assertEquals(ApplicationMetricsConstants.ENTITY_TYPE, entity.getEntityType());
        Assert.assertEquals(app.getApplicationId().toString(), entity.getEntityId());
        Assert.assertEquals(app.getName(), entity.getOtherInfo().get(ApplicationMetricsConstants.NAME_ENTITY_INFO));
        if (i != 1) {
            Assert.assertEquals(app.getQueue(), entity.getOtherInfo().get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO));
        }
        Assert.assertEquals(app.getApplicationSubmissionContext().getUnmanagedAM(), entity.getOtherInfo().get(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO));
        if (i != 1) {
            Assert.assertEquals(app.getApplicationSubmissionContext().getPriority().getPriority(), entity.getOtherInfo().get(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
        }
        Assert.assertEquals(app.getAmNodeLabelExpression(), entity.getOtherInfo().get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION));
        Assert.assertEquals(app.getApplicationSubmissionContext().getNodeLabelExpression(), entity.getOtherInfo().get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION));
        Assert.assertEquals(app.getUser(), entity.getOtherInfo().get(ApplicationMetricsConstants.USER_ENTITY_INFO));
        Assert.assertEquals(app.getApplicationType(), entity.getOtherInfo().get(ApplicationMetricsConstants.TYPE_ENTITY_INFO));
        Assert.assertEquals(app.getSubmitTime(), entity.getOtherInfo().get(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
        Assert.assertTrue(verifyAppTags(app.getApplicationTags(), entity.getOtherInfo()));
        if (i == 1) {
            Assert.assertEquals("uers1,user2", entity.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
            Assert.assertEquals(app.getApplicationSubmissionContext().getAMContainerSpec().getCommands(), entity.getOtherInfo().get(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND));
        } else {
            Assert.assertEquals("", entity.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
            Assert.assertEquals(app.getRMAppMetrics().getMemorySeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
            Assert.assertEquals(app.getRMAppMetrics().getVcoreSeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
            Assert.assertEquals(app.getRMAppMetrics().getPreemptedMemorySeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS).toString()));
            Assert.assertEquals(app.getRMAppMetrics().getPreemptedVcoreSeconds(), Long.parseLong(entity.getOtherInfo().get(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS).toString()));
        }
        Assert.assertEquals("context", entity.getOtherInfo().get(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT));
        boolean hasCreatedEvent = false;
        boolean hasUpdatedEvent = false;
        boolean hasFinishedEvent = false;
        boolean hasACLsUpdatedEvent = false;
        boolean hasStateUpdateEvent = false;
        for (TimelineEvent event : entity.getEvents()) {
            if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
                hasCreatedEvent = true;
                Assert.assertEquals(app.getStartTime(), event.getTimestamp());
            } else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
                hasFinishedEvent = true;
                Assert.assertEquals(app.getFinishTime(), event.getTimestamp());
                Assert.assertEquals(app.getDiagnostics().toString(), event.getEventInfo().get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
                Assert.assertEquals(app.getFinalApplicationStatus().toString(), event.getEventInfo().get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
                Assert.assertEquals(YarnApplicationState.FINISHED.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
            } else if (event.getEventType().equals(ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) {
                hasUpdatedEvent = true;
                Assert.assertEquals(4L, event.getTimestamp());
                if (1 == i) {
                    Assert.assertEquals(1, event.getEventInfo().get(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
                    Assert.assertEquals("new test queue", event.getEventInfo().get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO));
                }
            } else if (event.getEventType().equals(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
                hasACLsUpdatedEvent = true;
                Assert.assertEquals(4L, event.getTimestamp());
            } else if (event.getEventType().equals(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE)) {
                hasStateUpdateEvent = true;
                Assert.assertEquals(event.getTimestamp(), stateUpdateTimeStamp);
                Assert.assertEquals(YarnApplicationState.RUNNING.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
            }
        }
        // Do assertTrue verification separately for easier debug
        Assert.assertTrue(hasCreatedEvent);
        Assert.assertTrue(hasFinishedEvent);
        Assert.assertTrue(hasACLsUpdatedEvent);
        Assert.assertTrue(hasUpdatedEvent);
        Assert.assertTrue(hasStateUpdateEvent);
    }
}

Example 52 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestSystemMetricsPublisher method testPublishHostPortInfoOnContainerFinished.

@Test(timeout = 10000)
public void testPublishHostPortInfoOnContainerFinished() throws Exception {
    ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
    RMContainer container = createRMContainer(containerId);
    metricsPublisher.containerFinished(container, container.getFinishTime());
    TimelineEntity entity = null;
    do {
        entity = store.getEntity(containerId.toString(), ContainerMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class));
    } while (entity == null || entity.getEvents().size() < 1);
    Assert.assertNotNull(entity.getOtherInfo());
    Assert.assertEquals(2, entity.getOtherInfo().size());
    Assert.assertNotNull(entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_HOST_INFO));
    Assert.assertNotNull(entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PORT_INFO));
    Assert.assertEquals(container.getAllocatedNode().getHost(), entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_HOST_INFO));
    Assert.assertEquals(container.getAllocatedNode().getPort(), entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PORT_INFO));
}

Example 53 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestEntityGroupFSTimelineStore method testSummaryRead.

@Test
public void testSummaryRead() throws Exception {
    // Load data
    EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED);
    MutableCounterLong summaryLogEntityRead = store.metrics.getGetEntityToSummaryOps();
    long numEntityReadBefore = summaryLogEntityRead.value();
    TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithStore(config, store);
    appLogs.scanForLogs();
    appLogs.parseSummaryLogs(tdm);
    // Verify single entity read
    PluginStoreTestUtils.verifyTestEntities(tdm);
    // Verify multiple entities read
    TimelineEntities entities = tdm.getEntities("type_1", null, null, null, null, null, null, null, EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser());
    assertEquals(entities.getEntities().size(), 1);
    for (TimelineEntity entity : entities.getEntities()) {
        assertEquals((Long) 123L, entity.getStartTime());
    }
    // Verify metrics
    assertEquals(numEntityReadBefore + 5L, summaryLogEntityRead.value());
}

Example 54 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class PluginStoreTestUtils method writeEntities.

/**
   * Write timeline entities to a file system
   * @param entities
   * @param logPath
   * @param fs
   * @throws IOException
   */
static void writeEntities(TimelineEntities entities, Path logPath, FileSystem fs) throws IOException {
    FSDataOutputStream outStream = createLogFile(logPath, fs);
    JsonGenerator jsonGenerator = new JsonFactory().createGenerator(outStream);
    jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
    ObjectMapper objMapper = createObjectMapper();
    for (TimelineEntity entity : entities.getEntities()) {
        objMapper.writeValue(jsonGenerator, entity);
    }
    outStream.close();
}

Example 55 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project jstorm by alibaba.

the class JstormMaster method publishApplicationAttemptEvent.

private static void publishApplicationAttemptEvent(final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) {
    final TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(appAttemptId);
    entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
    entity.setDomainId(domainId);
    entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setEventType(appEvent.toString());
    event.setTimestamp(System.currentTimeMillis());
    entity.addEvent(event);
    try {
        timelineClient.putEntities(entity);
    } catch (YarnException e) {
        LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? JOYConstants.START : JOYConstants.END) + " event could not be published for " + appAttemptId.toString(), e);
    } catch (IOException e) {
        LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? JOYConstants.START : JOYConstants.END) + " event could not be published for " + appAttemptId.toString(), e);
    }
}