Examples with TimelineEvent org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent used on opensource projects

Example 36 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent in project hadoop by apache.

the class HBaseTimelineWriterImpl method storeEvents.

/**
   * Stores the events from the {@linkplain TimelineEvent} object.
   */
private void storeEvents(byte[] rowKey, Set<TimelineEvent> events, boolean isApplication) throws IOException {
    if (events != null) {
        for (TimelineEvent event : events) {
            if (event != null) {
                String eventId = event.getId();
                if (eventId != null) {
                    long eventTimestamp = event.getTimestamp();
                    // if the timestamp is not set, use the current timestamp
                    if (eventTimestamp == TimelineEvent.INVALID_TIMESTAMP) {
                        LOG.warn("timestamp is not set for event " + eventId + "! Using the current timestamp");
                        eventTimestamp = System.currentTimeMillis();
                    }
                    Map<String, Object> eventInfo = event.getInfo();
                    if ((eventInfo == null) || (eventInfo.size() == 0)) {
                        byte[] columnQualifierBytes = new EventColumnName(eventId, eventTimestamp, null).getColumnQualifier();
                        if (isApplication) {
                            ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable, columnQualifierBytes, null, Separator.EMPTY_BYTES);
                        } else {
                            EntityColumnPrefix.EVENT.store(rowKey, entityTable, columnQualifierBytes, null, Separator.EMPTY_BYTES);
                        }
                    } else {
                        for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
                            // eventId=infoKey
                            byte[] columnQualifierBytes = new EventColumnName(eventId, eventTimestamp, info.getKey()).getColumnQualifier();
                            if (isApplication) {
                                ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable, columnQualifierBytes, null, info.getValue());
                            } else {
                                EntityColumnPrefix.EVENT.store(rowKey, entityTable, columnQualifierBytes, null, info.getValue());
                            }
                        }
                    // for info: eventInfo
                    }
                }
            }
        }
    // event : events
    }
}

Example 37 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent in project hadoop by apache.

the class TestFlowDataGenerator method getEntityMaxEndTime.

static TimelineEntity getEntityMaxEndTime(long endTs) {
    TimelineEntity entity = new TimelineEntity();
    entity.setId("flowRunHello Max End time");
    entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
    TimelineEvent event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
    event.setTimestamp(endTs);
    String expKey = "foo_even_max_ finished";
    String expVal = "test_app_max_finished";
    event.addInfo(expKey, expVal);
    entity.addEvent(event);
    return entity;
}

Example 38 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent in project hadoop by apache.

the class TestFlowDataGenerator method getMaxFlushEntity.

static TimelineEntity getMaxFlushEntity(long startTs) {
    TimelineEntity entity = new TimelineEntity();
    String id = "flowRunHelloFlushEntityMax";
    String type = TimelineEntityType.YARN_APPLICATION.toString();
    entity.setId(id);
    entity.setType(type);
    entity.setCreatedTime(startTs);
    TimelineEvent event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
    event.setTimestamp(startTs + END_TS_INCR);
    entity.addEvent(event);
    return entity;
}

Example 39 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent in project hadoop by apache.

the class TestFlowDataGenerator method getEntityMinStartTime.

static TimelineEntity getEntityMinStartTime(long startTs) {
    TimelineEntity entity = new TimelineEntity();
    String id = "flowRunHelloMInStartTime";
    String type = TimelineEntityType.YARN_APPLICATION.toString();
    entity.setId(id);
    entity.setType(type);
    entity.setCreatedTime(startTs);
    TimelineEvent event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
    event.setTimestamp(startTs);
    entity.addEvent(event);
    return entity;
}

Example 40 with TimelineEvent

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent in project hadoop by apache.

the class TestFileSystemTimelineReaderImpl method loadEntityData.

private static void loadEntityData(String rootDir) throws Exception {
    File appDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1", "app");
    TimelineEntity entity11 = new TimelineEntity();
    entity11.setId("id_1");
    entity11.setType("app");
    entity11.setCreatedTime(1425016502000L);
    Map<String, Object> info1 = new HashMap<String, Object>();
    info1.put("info1", "val1");
    info1.put("info2", "val5");
    entity11.addInfo(info1);
    TimelineEvent event = new TimelineEvent();
    event.setId("event_1");
    event.setTimestamp(1425016502003L);
    entity11.addEvent(event);
    Set<TimelineMetric> metrics = new HashSet<TimelineMetric>();
    TimelineMetric metric1 = new TimelineMetric();
    metric1.setId("metric1");
    metric1.setType(TimelineMetric.Type.SINGLE_VALUE);
    metric1.addValue(1425016502006L, 113);
    metrics.add(metric1);
    TimelineMetric metric2 = new TimelineMetric();
    metric2.setId("metric2");
    metric2.setType(TimelineMetric.Type.TIME_SERIES);
    metric2.addValue(1425016502016L, 34);
    metrics.add(metric2);
    entity11.setMetrics(metrics);
    Map<String, String> configs = new HashMap<String, String>();
    configs.put("config_1", "127");
    entity11.setConfigs(configs);
    entity11.addRelatesToEntity("flow", "flow1");
    entity11.addIsRelatedToEntity("type1", "tid1_1");
    writeEntityFile(entity11, appDir);
    TimelineEntity entity12 = new TimelineEntity();
    entity12.setId("id_1");
    entity12.setType("app");
    configs.clear();
    configs.put("config_2", "23");
    configs.put("config_3", "abc");
    entity12.addConfigs(configs);
    metrics.clear();
    TimelineMetric metric12 = new TimelineMetric();
    metric12.setId("metric2");
    metric12.setType(TimelineMetric.Type.TIME_SERIES);
    metric12.addValue(1425016502032L, 48);
    metric12.addValue(1425016502054L, 51);
    metrics.add(metric12);
    TimelineMetric metric3 = new TimelineMetric();
    metric3.setId("metric3");
    metric3.setType(TimelineMetric.Type.SINGLE_VALUE);
    metric3.addValue(1425016502060L, 23L);
    metrics.add(metric3);
    entity12.setMetrics(metrics);
    entity12.addIsRelatedToEntity("type1", "tid1_2");
    entity12.addIsRelatedToEntity("type2", "tid2_1`");
    TimelineEvent event15 = new TimelineEvent();
    event15.setId("event_5");
    event15.setTimestamp(1425016502017L);
    entity12.addEvent(event15);
    writeEntityFile(entity12, appDir);
    TimelineEntity entity2 = new TimelineEntity();
    entity2.setId("id_2");
    entity2.setType("app");
    entity2.setCreatedTime(1425016501050L);
    Map<String, Object> info2 = new HashMap<String, Object>();
    info1.put("info2", 4);
    entity2.addInfo(info2);
    Map<String, String> configs2 = new HashMap<String, String>();
    configs2.put("config_1", "129");
    configs2.put("config_3", "def");
    entity2.setConfigs(configs2);
    TimelineEvent event2 = new TimelineEvent();
    event2.setId("event_2");
    event2.setTimestamp(1425016501003L);
    entity2.addEvent(event2);
    Set<TimelineMetric> metrics2 = new HashSet<TimelineMetric>();
    TimelineMetric metric21 = new TimelineMetric();
    metric21.setId("metric1");
    metric21.setType(TimelineMetric.Type.SINGLE_VALUE);
    metric21.addValue(1425016501006L, 300);
    metrics2.add(metric21);
    TimelineMetric metric22 = new TimelineMetric();
    metric22.setId("metric2");
    metric22.setType(TimelineMetric.Type.TIME_SERIES);
    metric22.addValue(1425016501056L, 31);
    metric22.addValue(1425016501084L, 70);
    metrics2.add(metric22);
    TimelineMetric metric23 = new TimelineMetric();
    metric23.setId("metric3");
    metric23.setType(TimelineMetric.Type.SINGLE_VALUE);
    metric23.addValue(1425016502060L, 23L);
    metrics2.add(metric23);
    entity2.setMetrics(metrics2);
    entity2.addRelatesToEntity("flow", "flow2");
    writeEntityFile(entity2, appDir);
    TimelineEntity entity3 = new TimelineEntity();
    entity3.setId("id_3");
    entity3.setType("app");
    entity3.setCreatedTime(1425016501050L);
    Map<String, Object> info3 = new HashMap<String, Object>();
    info3.put("info2", 3.5);
    info3.put("info4", 20);
    entity3.addInfo(info3);
    Map<String, String> configs3 = new HashMap<String, String>();
    configs3.put("config_1", "123");
    configs3.put("config_3", "abc");
    entity3.setConfigs(configs3);
    TimelineEvent event3 = new TimelineEvent();
    event3.setId("event_2");
    event3.setTimestamp(1425016501003L);
    entity3.addEvent(event3);
    TimelineEvent event4 = new TimelineEvent();
    event4.setId("event_4");
    event4.setTimestamp(1425016502006L);
    entity3.addEvent(event4);
    Set<TimelineMetric> metrics3 = new HashSet<TimelineMetric>();
    TimelineMetric metric31 = new TimelineMetric();
    metric31.setId("metric1");
    metric31.setType(TimelineMetric.Type.SINGLE_VALUE);
    metric31.addValue(1425016501006L, 124);
    metrics3.add(metric31);
    TimelineMetric metric32 = new TimelineMetric();
    metric32.setId("metric2");
    metric32.setType(TimelineMetric.Type.TIME_SERIES);
    metric32.addValue(1425016501056L, 31);
    metric32.addValue(1425016501084L, 74);
    metrics3.add(metric32);
    entity3.setMetrics(metrics3);
    entity3.addIsRelatedToEntity("type1", "tid1_2");
    writeEntityFile(entity3, appDir);
    TimelineEntity entity4 = new TimelineEntity();
    entity4.setId("id_4");
    entity4.setType("app");
    entity4.setCreatedTime(1425016502050L);
    TimelineEvent event44 = new TimelineEvent();
    event44.setId("event_4");
    event44.setTimestamp(1425016502003L);
    entity4.addEvent(event44);
    writeEntityFile(entity4, appDir);
    File attemptDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1", TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString());
    ApplicationAttemptEntity attempt1 = new ApplicationAttemptEntity();
    attempt1.setId("app-attempt-1");
    attempt1.setCreatedTime(1425017502003L);
    writeEntityFile(attempt1, attemptDir);
    ApplicationAttemptEntity attempt2 = new ApplicationAttemptEntity();
    attempt2.setId("app-attempt-2");
    attempt2.setCreatedTime(1425017502004L);
    writeEntityFile(attempt2, attemptDir);
    File entityDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1", TimelineEntityType.YARN_CONTAINER.toString());
    ContainerEntity containerEntity1 = new ContainerEntity();
    containerEntity1.setId("container_1_1");
    containerEntity1.setParent(attempt1.getIdentifier());
    containerEntity1.setCreatedTime(1425017502003L);
    writeEntityFile(containerEntity1, entityDir);
    ContainerEntity containerEntity2 = new ContainerEntity();
    containerEntity2.setId("container_2_1");
    containerEntity2.setParent(attempt2.getIdentifier());
    containerEntity2.setCreatedTime(1425018502003L);
    writeEntityFile(containerEntity2, entityDir);
    ContainerEntity containerEntity3 = new ContainerEntity();
    containerEntity3.setId("container_2_2");
    containerEntity3.setParent(attempt2.getIdentifier());
    containerEntity3.setCreatedTime(1425018502003L);
    writeEntityFile(containerEntity3, entityDir);
    File appDir2 = getAppDir(rootDir, "cluster1", "user1", "flow1,flow", "1", "app2", "app");
    TimelineEntity entity5 = new TimelineEntity();
    entity5.setId("id_5");
    entity5.setType("app");
    entity5.setCreatedTime(1425016502050L);
    writeEntityFile(entity5, appDir2);
}