Search in sources :

Example 41 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestTimelineWebServices method testGetEntityFields2.

@Test
public void testGetEntityFields2() throws Exception {
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("timeline").path("type_1").path("id_1").queryParam("fields", "lasteventonly," + "primaryfilters,relatedentities").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
    TimelineEntity entity = response.getEntity(TimelineEntity.class);
    Assert.assertNotNull(entity);
    Assert.assertEquals("id_1", entity.getEntityId());
    Assert.assertEquals("type_1", entity.getEntityType());
    Assert.assertEquals(123l, entity.getStartTime().longValue());
    Assert.assertEquals(1, entity.getEvents().size());
    Assert.assertEquals(4, entity.getPrimaryFilters().size());
    Assert.assertEquals(0, entity.getOtherInfo().size());
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) WebResource(com.sun.jersey.api.client.WebResource) TimelineEntity(org.apache.hadoop.yarn.api.records.timeline.TimelineEntity) Test(org.junit.Test)

Example 42 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestSystemMetricsPublisher method testPublishAppAttemptMetricsForUnmanagedAM.

@Test(timeout = 10000)
public void testPublishAppAttemptMetricsForUnmanagedAM() throws Exception {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
    RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, true);
    metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
    RMApp app = mock(RMApp.class);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
    TimelineEntity entity = null;
    do {
        entity = store.getEntity(appAttemptId.toString(), AppAttemptMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class));
    // ensure two events are both published before leaving the loop
    } while (entity == null || entity.getEvents().size() < 2);
}
Also used : RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) RMAppAttempt(org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) TimelineEntity(org.apache.hadoop.yarn.api.records.timeline.TimelineEntity) Test(org.junit.Test)

Example 43 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestSystemMetricsPublisher method testPublishAppAttemptMetrics.

@Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
    RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, false);
    metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
    RMApp app = mock(RMApp.class);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
    TimelineEntity entity = null;
    do {
        entity = store.getEntity(appAttemptId.toString(), AppAttemptMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class));
    // ensure two events are both published before leaving the loop
    } while (entity == null || entity.getEvents().size() < 2);
    // verify all the fields
    Assert.assertEquals(AppAttemptMetricsConstants.ENTITY_TYPE, entity.getEntityType());
    Assert.assertEquals(appAttemptId.toString(), entity.getEntityId());
    Assert.assertEquals(appAttemptId.getApplicationId().toString(), entity.getPrimaryFilters().get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator().next());
    boolean hasRegisteredEvent = false;
    boolean hasFinishedEvent = false;
    for (TimelineEvent event : entity.getEvents()) {
        if (event.getEventType().equals(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) {
            hasRegisteredEvent = true;
            Assert.assertEquals(appAttempt.getHost(), event.getEventInfo().get(AppAttemptMetricsConstants.HOST_INFO));
            Assert.assertEquals(appAttempt.getRpcPort(), event.getEventInfo().get(AppAttemptMetricsConstants.RPC_PORT_INFO));
            Assert.assertEquals(appAttempt.getMasterContainer().getId().toString(), event.getEventInfo().get(AppAttemptMetricsConstants.MASTER_CONTAINER_INFO));
        } else if (event.getEventType().equals(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) {
            hasFinishedEvent = true;
            Assert.assertEquals(appAttempt.getDiagnostics(), event.getEventInfo().get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO));
            Assert.assertEquals(appAttempt.getTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.TRACKING_URL_INFO));
            Assert.assertEquals(appAttempt.getOriginalTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO));
            Assert.assertEquals(FinalApplicationStatus.UNDEFINED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.FINAL_STATUS_INFO));
            Assert.assertEquals(YarnApplicationAttemptState.FINISHED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.STATE_INFO));
        }
    }
    Assert.assertTrue(hasRegisteredEvent && hasFinishedEvent);
}
Also used : TimelineEvent(org.apache.hadoop.yarn.api.records.timeline.TimelineEvent) RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) RMAppAttempt(org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) TimelineEntity(org.apache.hadoop.yarn.api.records.timeline.TimelineEntity) Test(org.junit.Test)

Example 44 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestSystemMetricsPublisher method testPublishContainerMetrics.

@Test(timeout = 10000)
public void testPublishContainerMetrics() throws Exception {
    ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
    RMContainer container = createRMContainer(containerId);
    metricsPublisher.containerCreated(container, container.getCreationTime());
    metricsPublisher.containerFinished(container, container.getFinishTime());
    TimelineEntity entity = null;
    do {
        entity = store.getEntity(containerId.toString(), ContainerMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class));
    // ensure two events are both published before leaving the loop
    } while (entity == null || entity.getEvents().size() < 2);
    // verify all the fields
    Assert.assertEquals(ContainerMetricsConstants.ENTITY_TYPE, entity.getEntityType());
    Assert.assertEquals(containerId.toString(), entity.getEntityId());
    Assert.assertEquals(containerId.getApplicationAttemptId().toString(), entity.getPrimaryFilters().get(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER).iterator().next());
    Assert.assertEquals(container.getAllocatedNode().getHost(), entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_HOST_INFO));
    Assert.assertEquals(container.getAllocatedNode().getPort(), entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PORT_INFO));
    Assert.assertEquals(container.getAllocatedResource().getMemorySize(), // variables for compare have same type.
    ((Integer) entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO)).longValue());
    Assert.assertEquals(container.getAllocatedResource().getVirtualCores(), entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_VCORE_INFO));
    Assert.assertEquals(container.getAllocatedPriority().getPriority(), entity.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO));
    boolean hasCreatedEvent = false;
    boolean hasFinishedEvent = false;
    for (TimelineEvent event : entity.getEvents()) {
        if (event.getEventType().equals(ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
            hasCreatedEvent = true;
            Assert.assertEquals(container.getCreationTime(), event.getTimestamp());
        } else if (event.getEventType().equals(ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
            hasFinishedEvent = true;
            Assert.assertEquals(container.getFinishTime(), event.getTimestamp());
            Assert.assertEquals(container.getDiagnosticsInfo(), event.getEventInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO));
            Assert.assertEquals(container.getContainerExitStatus(), event.getEventInfo().get(ContainerMetricsConstants.EXIT_STATUS_INFO));
            Assert.assertEquals(container.getContainerState().toString(), event.getEventInfo().get(ContainerMetricsConstants.STATE_INFO));
        }
    }
    Assert.assertTrue(hasCreatedEvent && hasFinishedEvent);
}
Also used : TimelineEvent(org.apache.hadoop.yarn.api.records.timeline.TimelineEvent) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) TimelineEntity(org.apache.hadoop.yarn.api.records.timeline.TimelineEntity) RMContainer(org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer) Test(org.junit.Test)

Example 45 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class JobHistoryEventHandler method processEventForTimelineServer.

private void processEventForTimelineServer(HistoryEvent event, JobId jobId, long timestamp) {
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setEventType(StringUtils.toUpperCase(event.getEventType().name()));
    tEvent.setTimestamp(timestamp);
    TimelineEntity tEntity = new TimelineEntity();
    switch(event.getEventType()) {
        case JOB_SUBMITTED:
            JobSubmittedEvent jse = (JobSubmittedEvent) event;
            tEvent.addEventInfo("SUBMIT_TIME", jse.getSubmitTime());
            tEvent.addEventInfo("QUEUE_NAME", jse.getJobQueueName());
            tEvent.addEventInfo("JOB_NAME", jse.getJobName());
            tEvent.addEventInfo("USER_NAME", jse.getUserName());
            tEvent.addEventInfo("JOB_CONF_PATH", jse.getJobConfPath());
            tEvent.addEventInfo("ACLS", jse.getJobAcls());
            tEvent.addEventInfo("JOB_QUEUE_NAME", jse.getJobQueueName());
            tEvent.addEventInfo("WORKFLOW_ID", jse.getWorkflowId());
            tEvent.addEventInfo("WORKFLOW_NAME", jse.getWorkflowName());
            tEvent.addEventInfo("WORKFLOW_NAME_NAME", jse.getWorkflowNodeName());
            tEvent.addEventInfo("WORKFLOW_ADJACENCIES", jse.getWorkflowAdjacencies());
            tEvent.addEventInfo("WORKFLOW_TAGS", jse.getWorkflowTags());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case JOB_STATUS_CHANGED:
            JobStatusChangedEvent jsce = (JobStatusChangedEvent) event;
            tEvent.addEventInfo("STATUS", jsce.getStatus());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case JOB_INFO_CHANGED:
            JobInfoChangeEvent jice = (JobInfoChangeEvent) event;
            tEvent.addEventInfo("SUBMIT_TIME", jice.getSubmitTime());
            tEvent.addEventInfo("LAUNCH_TIME", jice.getLaunchTime());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case JOB_INITED:
            JobInitedEvent jie = (JobInitedEvent) event;
            tEvent.addEventInfo("START_TIME", jie.getLaunchTime());
            tEvent.addEventInfo("STATUS", jie.getStatus());
            tEvent.addEventInfo("TOTAL_MAPS", jie.getTotalMaps());
            tEvent.addEventInfo("TOTAL_REDUCES", jie.getTotalReduces());
            tEvent.addEventInfo("UBERIZED", jie.getUberized());
            tEntity.setStartTime(jie.getLaunchTime());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case JOB_PRIORITY_CHANGED:
            JobPriorityChangeEvent jpce = (JobPriorityChangeEvent) event;
            tEvent.addEventInfo("PRIORITY", jpce.getPriority().toString());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case JOB_QUEUE_CHANGED:
            JobQueueChangeEvent jqe = (JobQueueChangeEvent) event;
            tEvent.addEventInfo("QUEUE_NAMES", jqe.getJobQueueName());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case JOB_FAILED:
        case JOB_KILLED:
        case JOB_ERROR:
            JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event;
            tEvent.addEventInfo("FINISH_TIME", juce.getFinishTime());
            tEvent.addEventInfo("NUM_MAPS", juce.getFinishedMaps());
            tEvent.addEventInfo("NUM_REDUCES", juce.getFinishedReduces());
            tEvent.addEventInfo("JOB_STATUS", juce.getStatus());
            tEvent.addEventInfo("DIAGNOSTICS", juce.getDiagnostics());
            tEvent.addEventInfo("FINISHED_MAPS", juce.getFinishedMaps());
            tEvent.addEventInfo("FINISHED_REDUCES", juce.getFinishedReduces());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case JOB_FINISHED:
            JobFinishedEvent jfe = (JobFinishedEvent) event;
            tEvent.addEventInfo("FINISH_TIME", jfe.getFinishTime());
            tEvent.addEventInfo("NUM_MAPS", jfe.getFinishedMaps());
            tEvent.addEventInfo("NUM_REDUCES", jfe.getFinishedReduces());
            tEvent.addEventInfo("FAILED_MAPS", jfe.getFailedMaps());
            tEvent.addEventInfo("FAILED_REDUCES", jfe.getFailedReduces());
            tEvent.addEventInfo("FINISHED_MAPS", jfe.getFinishedMaps());
            tEvent.addEventInfo("FINISHED_REDUCES", jfe.getFinishedReduces());
            tEvent.addEventInfo("MAP_COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(jfe.getMapCounters()));
            tEvent.addEventInfo("REDUCE_COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(jfe.getReduceCounters()));
            tEvent.addEventInfo("TOTAL_COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(jfe.getTotalCounters()));
            tEvent.addEventInfo("JOB_STATUS", JobState.SUCCEEDED.toString());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        case TASK_STARTED:
            TaskStartedEvent tse = (TaskStartedEvent) event;
            tEvent.addEventInfo("TASK_TYPE", tse.getTaskType().toString());
            tEvent.addEventInfo("START_TIME", tse.getStartTime());
            tEvent.addEventInfo("SPLIT_LOCATIONS", tse.getSplitLocations());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(tse.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case TASK_FAILED:
            TaskFailedEvent tfe = (TaskFailedEvent) event;
            tEvent.addEventInfo("TASK_TYPE", tfe.getTaskType().toString());
            tEvent.addEventInfo("STATUS", TaskStatus.State.FAILED.toString());
            tEvent.addEventInfo("FINISH_TIME", tfe.getFinishTime());
            tEvent.addEventInfo("ERROR", tfe.getError());
            tEvent.addEventInfo("FAILED_ATTEMPT_ID", tfe.getFailedAttemptID() == null ? "" : tfe.getFailedAttemptID().toString());
            tEvent.addEventInfo("COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(tfe.getCounters()));
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(tfe.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case TASK_UPDATED:
            TaskUpdatedEvent tue = (TaskUpdatedEvent) event;
            tEvent.addEventInfo("FINISH_TIME", tue.getFinishTime());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(tue.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case TASK_FINISHED:
            TaskFinishedEvent tfe2 = (TaskFinishedEvent) event;
            tEvent.addEventInfo("TASK_TYPE", tfe2.getTaskType().toString());
            tEvent.addEventInfo("COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(tfe2.getCounters()));
            tEvent.addEventInfo("FINISH_TIME", tfe2.getFinishTime());
            tEvent.addEventInfo("STATUS", TaskStatus.State.SUCCEEDED.toString());
            tEvent.addEventInfo("SUCCESSFUL_TASK_ATTEMPT_ID", tfe2.getSuccessfulTaskAttemptId() == null ? "" : tfe2.getSuccessfulTaskAttemptId().toString());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(tfe2.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case MAP_ATTEMPT_STARTED:
        case CLEANUP_ATTEMPT_STARTED:
        case REDUCE_ATTEMPT_STARTED:
        case SETUP_ATTEMPT_STARTED:
            TaskAttemptStartedEvent tase = (TaskAttemptStartedEvent) event;
            tEvent.addEventInfo("TASK_TYPE", tase.getTaskType().toString());
            tEvent.addEventInfo("TASK_ATTEMPT_ID", tase.getTaskAttemptId().toString());
            tEvent.addEventInfo("START_TIME", tase.getStartTime());
            tEvent.addEventInfo("HTTP_PORT", tase.getHttpPort());
            tEvent.addEventInfo("TRACKER_NAME", tase.getTrackerName());
            tEvent.addEventInfo("SHUFFLE_PORT", tase.getShufflePort());
            tEvent.addEventInfo("CONTAINER_ID", tase.getContainerId() == null ? "" : tase.getContainerId().toString());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(tase.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case MAP_ATTEMPT_FAILED:
        case CLEANUP_ATTEMPT_FAILED:
        case REDUCE_ATTEMPT_FAILED:
        case SETUP_ATTEMPT_FAILED:
        case MAP_ATTEMPT_KILLED:
        case CLEANUP_ATTEMPT_KILLED:
        case REDUCE_ATTEMPT_KILLED:
        case SETUP_ATTEMPT_KILLED:
            TaskAttemptUnsuccessfulCompletionEvent tauce = (TaskAttemptUnsuccessfulCompletionEvent) event;
            tEvent.addEventInfo("TASK_TYPE", tauce.getTaskType().toString());
            tEvent.addEventInfo("TASK_ATTEMPT_ID", tauce.getTaskAttemptId() == null ? "" : tauce.getTaskAttemptId().toString());
            tEvent.addEventInfo("FINISH_TIME", tauce.getFinishTime());
            tEvent.addEventInfo("ERROR", tauce.getError());
            tEvent.addEventInfo("STATUS", tauce.getTaskStatus());
            tEvent.addEventInfo("HOSTNAME", tauce.getHostname());
            tEvent.addEventInfo("PORT", tauce.getPort());
            tEvent.addEventInfo("RACK_NAME", tauce.getRackName());
            tEvent.addEventInfo("SHUFFLE_FINISH_TIME", tauce.getFinishTime());
            tEvent.addEventInfo("SORT_FINISH_TIME", tauce.getFinishTime());
            tEvent.addEventInfo("MAP_FINISH_TIME", tauce.getFinishTime());
            tEvent.addEventInfo("COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(tauce.getCounters()));
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(tauce.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case MAP_ATTEMPT_FINISHED:
            MapAttemptFinishedEvent mafe = (MapAttemptFinishedEvent) event;
            tEvent.addEventInfo("TASK_TYPE", mafe.getTaskType().toString());
            tEvent.addEventInfo("FINISH_TIME", mafe.getFinishTime());
            tEvent.addEventInfo("STATUS", mafe.getTaskStatus());
            tEvent.addEventInfo("STATE", mafe.getState());
            tEvent.addEventInfo("MAP_FINISH_TIME", mafe.getMapFinishTime());
            tEvent.addEventInfo("COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(mafe.getCounters()));
            tEvent.addEventInfo("HOSTNAME", mafe.getHostname());
            tEvent.addEventInfo("PORT", mafe.getPort());
            tEvent.addEventInfo("RACK_NAME", mafe.getRackName());
            tEvent.addEventInfo("ATTEMPT_ID", mafe.getAttemptId() == null ? "" : mafe.getAttemptId().toString());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(mafe.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case REDUCE_ATTEMPT_FINISHED:
            ReduceAttemptFinishedEvent rafe = (ReduceAttemptFinishedEvent) event;
            tEvent.addEventInfo("TASK_TYPE", rafe.getTaskType().toString());
            tEvent.addEventInfo("ATTEMPT_ID", rafe.getAttemptId() == null ? "" : rafe.getAttemptId().toString());
            tEvent.addEventInfo("FINISH_TIME", rafe.getFinishTime());
            tEvent.addEventInfo("STATUS", rafe.getTaskStatus());
            tEvent.addEventInfo("STATE", rafe.getState());
            tEvent.addEventInfo("SHUFFLE_FINISH_TIME", rafe.getShuffleFinishTime());
            tEvent.addEventInfo("SORT_FINISH_TIME", rafe.getSortFinishTime());
            tEvent.addEventInfo("COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(rafe.getCounters()));
            tEvent.addEventInfo("HOSTNAME", rafe.getHostname());
            tEvent.addEventInfo("PORT", rafe.getPort());
            tEvent.addEventInfo("RACK_NAME", rafe.getRackName());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(rafe.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case SETUP_ATTEMPT_FINISHED:
        case CLEANUP_ATTEMPT_FINISHED:
            TaskAttemptFinishedEvent tafe = (TaskAttemptFinishedEvent) event;
            tEvent.addEventInfo("TASK_TYPE", tafe.getTaskType().toString());
            tEvent.addEventInfo("ATTEMPT_ID", tafe.getAttemptId() == null ? "" : tafe.getAttemptId().toString());
            tEvent.addEventInfo("FINISH_TIME", tafe.getFinishTime());
            tEvent.addEventInfo("STATUS", tafe.getTaskStatus());
            tEvent.addEventInfo("STATE", tafe.getState());
            tEvent.addEventInfo("COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(tafe.getCounters()));
            tEvent.addEventInfo("HOSTNAME", tafe.getHostname());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(tafe.getTaskId().toString());
            tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
            tEntity.addRelatedEntity(MAPREDUCE_JOB_ENTITY_TYPE, jobId.toString());
            break;
        case AM_STARTED:
            AMStartedEvent ase = (AMStartedEvent) event;
            tEvent.addEventInfo("APPLICATION_ATTEMPT_ID", ase.getAppAttemptId() == null ? "" : ase.getAppAttemptId().toString());
            tEvent.addEventInfo("CONTAINER_ID", ase.getContainerId() == null ? "" : ase.getContainerId().toString());
            tEvent.addEventInfo("NODE_MANAGER_HOST", ase.getNodeManagerHost());
            tEvent.addEventInfo("NODE_MANAGER_PORT", ase.getNodeManagerPort());
            tEvent.addEventInfo("NODE_MANAGER_HTTP_PORT", ase.getNodeManagerHttpPort());
            tEvent.addEventInfo("START_TIME", ase.getStartTime());
            tEvent.addEventInfo("SUBMIT_TIME", ase.getSubmitTime());
            tEntity.addEvent(tEvent);
            tEntity.setEntityId(jobId.toString());
            tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE);
            break;
        default:
            break;
    }
    try {
        TimelinePutResponse response = timelineClient.putEntities(tEntity);
        List<TimelinePutResponse.TimelinePutError> errors = response.getErrors();
        if (errors.size() == 0) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Timeline entities are successfully put in event " + event.getEventType());
            }
        } else {
            for (TimelinePutResponse.TimelinePutError error : errors) {
                LOG.error("Error when publishing entity [" + error.getEntityType() + "," + error.getEntityId() + "], server side error code: " + error.getErrorCode());
            }
        }
    } catch (YarnException | IOException | ClientHandlerException ex) {
        LOG.error("Error putting entity " + tEntity.getEntityId() + " to Timeline" + "Server", ex);
    }
}
Also used : TimelineEvent(org.apache.hadoop.yarn.api.records.timeline.TimelineEvent) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) ClientHandlerException(com.sun.jersey.api.client.ClientHandlerException) TimelinePutResponse(org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse) IOException(java.io.IOException) TimelineEntity(org.apache.hadoop.yarn.api.records.timeline.TimelineEntity)

Aggregations

TimelineEntity (org.apache.hadoop.yarn.api.records.timeline.TimelineEntity)148 TimelineEvent (org.apache.hadoop.yarn.api.records.timeline.TimelineEvent)66 Test (org.junit.Test)61 TimelineEntities (org.apache.hadoop.yarn.api.records.timeline.TimelineEntities)30 HashMap (java.util.HashMap)23 TimelinePutResponse (org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse)21 IOException (java.io.IOException)17 YarnException (org.apache.hadoop.yarn.exceptions.YarnException)16 ClientResponse (com.sun.jersey.api.client.ClientResponse)13 WebResource (com.sun.jersey.api.client.WebResource)12 Set (java.util.Set)12 Configuration (org.apache.hadoop.conf.Configuration)11 HashSet (java.util.HashSet)8 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)8 Map (java.util.Map)7 Path (org.apache.hadoop.fs.Path)7 Field (org.apache.hadoop.yarn.server.timeline.TimelineReader.Field)6 DAGHistoryEvent (org.apache.tez.dag.history.DAGHistoryEvent)6 TezDAGID (org.apache.tez.dag.records.TezDAGID)6 ArrayList (java.util.ArrayList)5