Examples with TimelineEvent org.apache.hadoop.yarn.api.records.timeline.TimelineEvent used on opensource projects

Example 16 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEvent in project hadoop by apache.

the class KeyValueBasedTimelineStore method put.

@Override
public synchronized TimelinePutResponse put(TimelineEntities data) {
    TimelinePutResponse response = new TimelinePutResponse();
    if (getServiceStopped()) {
        LOG.info("Service stopped, return null for the storage");
        TimelinePutError error = new TimelinePutError();
        error.setErrorCode(TimelinePutError.IO_EXCEPTION);
        response.addError(error);
        return response;
    }
    for (TimelineEntity entity : data.getEntities()) {
        EntityIdentifier entityId = new EntityIdentifier(entity.getEntityId(), entity.getEntityType());
        // store entity info in memory
        TimelineEntity existingEntity = entities.get(entityId);
        boolean needsPut = false;
        if (existingEntity == null) {
            existingEntity = new TimelineEntity();
            existingEntity.setEntityId(entity.getEntityId());
            existingEntity.setEntityType(entity.getEntityType());
            existingEntity.setStartTime(entity.getStartTime());
            if (entity.getDomainId() == null || entity.getDomainId().length() == 0) {
                TimelinePutError error = new TimelinePutError();
                error.setEntityId(entityId.getId());
                error.setEntityType(entityId.getType());
                error.setErrorCode(TimelinePutError.NO_DOMAIN);
                response.addError(error);
                continue;
            }
            existingEntity.setDomainId(entity.getDomainId());
            // insert a new entity to the storage, update insert time map
            entityInsertTimes.put(entityId, System.currentTimeMillis());
            needsPut = true;
        }
        if (entity.getEvents() != null) {
            if (existingEntity.getEvents() == null) {
                existingEntity.setEvents(entity.getEvents());
            } else {
                existingEntity.addEvents(entity.getEvents());
            }
            Collections.sort(existingEntity.getEvents());
            needsPut = true;
        }
        // check startTime
        if (existingEntity.getStartTime() == null) {
            if (existingEntity.getEvents() == null || existingEntity.getEvents().isEmpty()) {
                TimelinePutError error = new TimelinePutError();
                error.setEntityId(entityId.getId());
                error.setEntityType(entityId.getType());
                error.setErrorCode(TimelinePutError.NO_START_TIME);
                response.addError(error);
                entities.remove(entityId);
                entityInsertTimes.remove(entityId);
                continue;
            } else {
                Long min = Long.MAX_VALUE;
                for (TimelineEvent e : entity.getEvents()) {
                    if (min > e.getTimestamp()) {
                        min = e.getTimestamp();
                    }
                }
                existingEntity.setStartTime(min);
                needsPut = true;
            }
        }
        if (entity.getPrimaryFilters() != null) {
            if (existingEntity.getPrimaryFilters() == null) {
                existingEntity.setPrimaryFilters(new HashMap<String, Set<Object>>());
            }
            for (Entry<String, Set<Object>> pf : entity.getPrimaryFilters().entrySet()) {
                for (Object pfo : pf.getValue()) {
                    existingEntity.addPrimaryFilter(pf.getKey(), KeyValueBasedTimelineStoreUtils.compactNumber(pfo));
                    needsPut = true;
                }
            }
        }
        if (entity.getOtherInfo() != null) {
            if (existingEntity.getOtherInfo() == null) {
                existingEntity.setOtherInfo(new HashMap<String, Object>());
            }
            for (Entry<String, Object> info : entity.getOtherInfo().entrySet()) {
                existingEntity.addOtherInfo(info.getKey(), KeyValueBasedTimelineStoreUtils.compactNumber(info.getValue()));
                needsPut = true;
            }
        }
        if (needsPut) {
            entities.put(entityId, existingEntity);
        }
        // relate it to other entities
        if (entity.getRelatedEntities() == null) {
            continue;
        }
        for (Entry<String, Set<String>> partRelatedEntities : entity.getRelatedEntities().entrySet()) {
            if (partRelatedEntities == null) {
                continue;
            }
            for (String idStr : partRelatedEntities.getValue()) {
                EntityIdentifier relatedEntityId = new EntityIdentifier(idStr, partRelatedEntities.getKey());
                TimelineEntity relatedEntity = entities.get(relatedEntityId);
                if (relatedEntity != null) {
                    if (relatedEntity.getDomainId().equals(existingEntity.getDomainId())) {
                        relatedEntity.addRelatedEntity(existingEntity.getEntityType(), existingEntity.getEntityId());
                        entities.put(relatedEntityId, relatedEntity);
                    } else {
                        // in this case the entity will be put, but the relation will be
                        // ignored
                        TimelinePutError error = new TimelinePutError();
                        error.setEntityType(existingEntity.getEntityType());
                        error.setEntityId(existingEntity.getEntityId());
                        error.setErrorCode(TimelinePutError.FORBIDDEN_RELATION);
                        response.addError(error);
                    }
                } else {
                    relatedEntity = new TimelineEntity();
                    relatedEntity.setEntityId(relatedEntityId.getId());
                    relatedEntity.setEntityType(relatedEntityId.getType());
                    relatedEntity.setStartTime(existingEntity.getStartTime());
                    relatedEntity.addRelatedEntity(existingEntity.getEntityType(), existingEntity.getEntityId());
                    relatedEntity.setDomainId(existingEntity.getDomainId());
                    entities.put(relatedEntityId, relatedEntity);
                    entityInsertTimes.put(relatedEntityId, System.currentTimeMillis());
                }
            }
        }
    }
    return response;
}

Example 17 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEvent in project hadoop by apache.

the class RollingLevelDBTimelineStore method getEntity.

/**
   * Read entity from a db iterator. If no information is found in the specified
   * fields for this entity, return null.
   */
private static TimelineEntity getEntity(String entityId, String entityType, Long startTime, EnumSet<Field> fields, DBIterator iterator, byte[] prefix, int prefixlen) throws IOException {
    if (fields == null) {
        fields = EnumSet.allOf(Field.class);
    }
    TimelineEntity entity = new TimelineEntity();
    boolean events = false;
    boolean lastEvent = false;
    if (fields.contains(Field.EVENTS)) {
        events = true;
    } else if (fields.contains(Field.LAST_EVENT_ONLY)) {
        lastEvent = true;
    } else {
        entity.setEvents(null);
    }
    boolean relatedEntities = false;
    if (fields.contains(Field.RELATED_ENTITIES)) {
        relatedEntities = true;
    } else {
        entity.setRelatedEntities(null);
    }
    boolean primaryFilters = false;
    if (fields.contains(Field.PRIMARY_FILTERS)) {
        primaryFilters = true;
    } else {
        entity.setPrimaryFilters(null);
    }
    boolean otherInfo = false;
    if (fields.contains(Field.OTHER_INFO)) {
        otherInfo = true;
    } else {
        entity.setOtherInfo(null);
    }
    // of a requested field
    for (; iterator.hasNext(); iterator.next()) {
        byte[] key = iterator.peekNext().getKey();
        if (!prefixMatches(prefix, prefixlen, key)) {
            break;
        }
        if (key.length == prefixlen) {
            continue;
        }
        if (key[prefixlen] == PRIMARY_FILTERS_COLUMN[0]) {
            if (primaryFilters) {
                addPrimaryFilter(entity, key, prefixlen + PRIMARY_FILTERS_COLUMN.length);
            }
        } else if (key[prefixlen] == OTHER_INFO_COLUMN[0]) {
            if (otherInfo) {
                entity.addOtherInfo(parseRemainingKey(key, prefixlen + OTHER_INFO_COLUMN.length), fstConf.asObject(iterator.peekNext().getValue()));
            }
        } else if (key[prefixlen] == RELATED_ENTITIES_COLUMN[0]) {
            if (relatedEntities) {
                addRelatedEntity(entity, key, prefixlen + RELATED_ENTITIES_COLUMN.length);
            }
        } else if (key[prefixlen] == EVENTS_COLUMN[0]) {
            if (events || (lastEvent && entity.getEvents().size() == 0)) {
                TimelineEvent event = getEntityEvent(null, key, prefixlen + EVENTS_COLUMN.length, iterator.peekNext().getValue());
                if (event != null) {
                    entity.addEvent(event);
                }
            }
        } else if (key[prefixlen] == DOMAIN_ID_COLUMN[0]) {
            byte[] v = iterator.peekNext().getValue();
            String domainId = new String(v, UTF_8);
            entity.setDomainId(domainId);
        } else {
            LOG.warn(String.format("Found unexpected column for entity %s of " + "type %s (0x%02x)", entityId, entityType, key[prefixlen]));
        }
    }
    entity.setEntityId(entityId);
    entity.setEntityType(entityType);
    entity.setStartTime(startTime);
    return entity;
}

Example 18 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEvent in project hadoop by apache.

the class ApplicationMaster method publishContainerEndEvent.

@VisibleForTesting
void publishContainerEndEvent(final TimelineClient timelineClient, ContainerStatus container, String domainId, UserGroupInformation ugi) {
    final TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(container.getContainerId().toString());
    entity.setEntityType(DSEntity.DS_CONTAINER.toString());
    entity.setDomainId(domainId);
    entity.addPrimaryFilter(USER_TIMELINE_FILTER_NAME, ugi.getShortUserName());
    entity.addPrimaryFilter(APPID_TIMELINE_FILTER_NAME, container.getContainerId().getApplicationAttemptId().getApplicationId().toString());
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType(DSEvent.DS_CONTAINER_END.toString());
    event.addEventInfo("State", container.getState().name());
    event.addEventInfo("Exit Status", container.getExitStatus());
    entity.addEvent(event);
    try {
        processTimelineResponseErrors(putContainerEntity(timelineClient, container.getContainerId().getApplicationAttemptId(), entity));
    } catch (YarnException | IOException | ClientHandlerException e) {
        LOG.error("Container end event could not be published for " + container.getContainerId().toString(), e);
    }
}

Example 19 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEvent in project hadoop by apache.

the class TestTimelineWebServicesWithSSL method testPutEntities.

@Test
public void testPutEntities() throws Exception {
    TestTimelineClient client = new TestTimelineClient();
    try {
        client.init(conf);
        client.start();
        TimelineEntity expectedEntity = new TimelineEntity();
        expectedEntity.setEntityType("test entity type");
        expectedEntity.setEntityId("test entity id");
        expectedEntity.setDomainId("test domain id");
        TimelineEvent event = new TimelineEvent();
        event.setEventType("test event type");
        event.setTimestamp(0L);
        expectedEntity.addEvent(event);
        TimelinePutResponse response = client.putEntities(expectedEntity);
        Assert.assertEquals(0, response.getErrors().size());
        Assert.assertTrue(client.resp.toString().contains("https"));
        TimelineEntity actualEntity = store.getEntity(expectedEntity.getEntityId(), expectedEntity.getEntityType(), EnumSet.allOf(Field.class));
        Assert.assertNotNull(actualEntity);
        Assert.assertEquals(expectedEntity.getEntityId(), actualEntity.getEntityId());
        Assert.assertEquals(expectedEntity.getEntityType(), actualEntity.getEntityType());
    } finally {
        client.stop();
        client.close();
    }
}

Example 20 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEvent in project hadoop by apache.

the class TestSystemMetricsPublisher method testPublishAppAttemptMetrics.

@Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
    RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, false);
    metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
    RMApp app = mock(RMApp.class);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
    TimelineEntity entity = null;
    do {
        entity = store.getEntity(appAttemptId.toString(), AppAttemptMetricsConstants.ENTITY_TYPE, EnumSet.allOf(Field.class));
    // ensure two events are both published before leaving the loop
    } while (entity == null || entity.getEvents().size() < 2);
    // verify all the fields
    Assert.assertEquals(AppAttemptMetricsConstants.ENTITY_TYPE, entity.getEntityType());
    Assert.assertEquals(appAttemptId.toString(), entity.getEntityId());
    Assert.assertEquals(appAttemptId.getApplicationId().toString(), entity.getPrimaryFilters().get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator().next());
    boolean hasRegisteredEvent = false;
    boolean hasFinishedEvent = false;
    for (TimelineEvent event : entity.getEvents()) {
        if (event.getEventType().equals(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) {
            hasRegisteredEvent = true;
            Assert.assertEquals(appAttempt.getHost(), event.getEventInfo().get(AppAttemptMetricsConstants.HOST_INFO));
            Assert.assertEquals(appAttempt.getRpcPort(), event.getEventInfo().get(AppAttemptMetricsConstants.RPC_PORT_INFO));
            Assert.assertEquals(appAttempt.getMasterContainer().getId().toString(), event.getEventInfo().get(AppAttemptMetricsConstants.MASTER_CONTAINER_INFO));
        } else if (event.getEventType().equals(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) {
            hasFinishedEvent = true;
            Assert.assertEquals(appAttempt.getDiagnostics(), event.getEventInfo().get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO));
            Assert.assertEquals(appAttempt.getTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.TRACKING_URL_INFO));
            Assert.assertEquals(appAttempt.getOriginalTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO));
            Assert.assertEquals(FinalApplicationStatus.UNDEFINED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.FINAL_STATUS_INFO));
            Assert.assertEquals(YarnApplicationAttemptState.FINISHED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.STATE_INFO));
        }
    }
    Assert.assertTrue(hasRegisteredEvent && hasFinishedEvent);
}