Examples with EventColumnName org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName used on opensource projects

Example 1 with EventColumnName

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName in project hadoop by apache.

the class TestHBaseTimelineStorageEntities method testEventsWithEmptyInfo.

@Test
public void testEventsWithEmptyInfo() throws IOException {
    TimelineEvent event = new TimelineEvent();
    String eventId = "foo_ev e  nt_id";
    event.setId(eventId);
    Long expTs = 1436512802000L;
    event.setTimestamp(expTs);
    final TimelineEntity entity = new TimelineEntity();
    entity.setId("attempt_1329348432655_0001_m_000008_18");
    entity.setType("FOO_ATTEMPT");
    entity.addEvent(event);
    TimelineEntities entities = new TimelineEntities();
    entities.addEntity(entity);
    HBaseTimelineWriterImpl hbi = null;
    try {
        Configuration c1 = util.getConfiguration();
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.start();
        String cluster = "cluster_test_empty_eventkey";
        String user = "user_emptyeventkey";
        String flow = "other_flow_name";
        String flowVersion = "1111F01C2287BA";
        long runid = 1009876543218L;
        String appName = ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1).toString();
        byte[] startRow = new EntityRowKeyPrefix(cluster, user, flow, runid, appName).getRowKeyPrefix();
        hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
        hbi.stop();
        // scan the table and see that entity exists
        Scan s = new Scan();
        s.setStartRow(startRow);
        s.addFamily(EntityColumnFamily.INFO.getBytes());
        Connection conn = ConnectionFactory.createConnection(c1);
        ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
        int rowCount = 0;
        for (Result result : scanner) {
            if (result != null && !result.isEmpty()) {
                rowCount++;
                // check the row key
                byte[] row1 = result.getRow();
                assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName, entity));
                Map<EventColumnName, Object> eventsResult = EntityColumnPrefix.EVENT.readResults(result, new EventColumnNameConverter());
                // there should be only one event
                assertEquals(1, eventsResult.size());
                for (Map.Entry<EventColumnName, Object> e : eventsResult.entrySet()) {
                    EventColumnName eventColumnName = e.getKey();
                    // the qualifier is a compound key
                    // hence match individual values
                    assertEquals(eventId, eventColumnName.getId());
                    assertEquals(expTs, eventColumnName.getTimestamp());
                    // key must be empty
                    assertNull(eventColumnName.getInfoKey());
                    Object value = e.getValue();
                    // value should be empty
                    assertEquals("", value.toString());
                }
            }
        }
        assertEquals(1, rowCount);
        // read the timeline entity using the reader this time
        TimelineEntity e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        Set<TimelineEntity> es1 = reader.getEntities(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        assertNotNull(e1);
        assertEquals(1, es1.size());
        // check the events
        NavigableSet<TimelineEvent> events = e1.getEvents();
        // there should be only one event
        assertEquals(1, events.size());
        for (TimelineEvent e : events) {
            assertEquals(eventId, e.getId());
            assertEquals(expTs, Long.valueOf(e.getTimestamp()));
            Map<String, Object> info = e.getInfo();
            assertTrue(info == null || info.isEmpty());
        }
    } finally {
        if (hbi != null) {
            hbi.stop();
            hbi.close();
        }
    }
}

Example 2 with EventColumnName

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testEvents.

@Test
public void testEvents() throws IOException {
    TimelineEvent event = new TimelineEvent();
    String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
    event.setId(eventId);
    Long expTs = 1436512802000L;
    event.setTimestamp(expTs);
    String expKey = "foo_event";
    Object expVal = "test";
    event.addInfo(expKey, expVal);
    final TimelineEntity entity = new ApplicationEntity();
    entity.setId(ApplicationId.newInstance(0, 1).toString());
    entity.addEvent(event);
    TimelineEntities entities = new TimelineEntities();
    entities.addEntity(entity);
    HBaseTimelineWriterImpl hbi = null;
    try {
        Configuration c1 = util.getConfiguration();
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.start();
        String cluster = "cluster_test_events";
        String user = "user2";
        String flow = "other_flow_name";
        String flowVersion = "1111F01C2287BA";
        long runid = 1009876543218L;
        String appName = "application_123465899910_1001";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
        hbi.stop();
        // retrieve the row
        ApplicationRowKey applicationRowKey = new ApplicationRowKey(cluster, user, flow, runid, appName);
        byte[] rowKey = applicationRowKey.getRowKey();
        Get get = new Get(rowKey);
        get.setMaxVersions(Integer.MAX_VALUE);
        Connection conn = ConnectionFactory.createConnection(c1);
        Result result = new ApplicationTable().getResult(c1, conn, get);
        assertTrue(result != null);
        // check the row key
        byte[] row1 = result.getRow();
        assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid, appName));
        Map<EventColumnName, Object> eventsResult = ApplicationColumnPrefix.EVENT.readResults(result, new EventColumnNameConverter());
        // there should be only one event
        assertEquals(1, eventsResult.size());
        for (Map.Entry<EventColumnName, Object> e : eventsResult.entrySet()) {
            EventColumnName eventColumnName = e.getKey();
            // the qualifier is a compound key
            // hence match individual values
            assertEquals(eventId, eventColumnName.getId());
            assertEquals(expTs, eventColumnName.getTimestamp());
            assertEquals(expKey, eventColumnName.getInfoKey());
            Object value = e.getValue();
            // there should be only one timestamp and value
            assertEquals(expVal, value.toString());
        }
        // read the timeline entity using the reader this time
        TimelineEntity e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        TimelineEntity e2 = reader.getEntity(new TimelineReaderContext(cluster, user, null, null, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        assertNotNull(e1);
        assertNotNull(e2);
        assertEquals(e1, e2);
        // check the events
        NavigableSet<TimelineEvent> events = e1.getEvents();
        // there should be only one event
        assertEquals(1, events.size());
        for (TimelineEvent e : events) {
            assertEquals(eventId, e.getId());
            assertEquals(expTs, Long.valueOf(e.getTimestamp()));
            Map<String, Object> info = e.getInfo();
            assertEquals(1, info.size());
            for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
                assertEquals(expKey, infoEntry.getKey());
                assertEquals(expVal, infoEntry.getValue());
            }
        }
    } finally {
        if (hbi != null) {
            hbi.stop();
            hbi.close();
        }
    }
}

Example 3 with EventColumnName

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName in project hadoop by apache.

the class HBaseTimelineWriterImpl method storeEvents.

/**
   * Stores the events from the {@linkplain TimelineEvent} object.
   */
private void storeEvents(byte[] rowKey, Set<TimelineEvent> events, boolean isApplication) throws IOException {
    if (events != null) {
        for (TimelineEvent event : events) {
            if (event != null) {
                String eventId = event.getId();
                if (eventId != null) {
                    long eventTimestamp = event.getTimestamp();
                    // if the timestamp is not set, use the current timestamp
                    if (eventTimestamp == TimelineEvent.INVALID_TIMESTAMP) {
                        LOG.warn("timestamp is not set for event " + eventId + "! Using the current timestamp");
                        eventTimestamp = System.currentTimeMillis();
                    }
                    Map<String, Object> eventInfo = event.getInfo();
                    if ((eventInfo == null) || (eventInfo.size() == 0)) {
                        byte[] columnQualifierBytes = new EventColumnName(eventId, eventTimestamp, null).getColumnQualifier();
                        if (isApplication) {
                            ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable, columnQualifierBytes, null, Separator.EMPTY_BYTES);
                        } else {
                            EntityColumnPrefix.EVENT.store(rowKey, entityTable, columnQualifierBytes, null, Separator.EMPTY_BYTES);
                        }
                    } else {
                        for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
                            // eventId=infoKey
                            byte[] columnQualifierBytes = new EventColumnName(eventId, eventTimestamp, info.getKey()).getColumnQualifier();
                            if (isApplication) {
                                ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable, columnQualifierBytes, null, info.getValue());
                            } else {
                                EntityColumnPrefix.EVENT.store(rowKey, entityTable, columnQualifierBytes, null, info.getValue());
                            }
                        }
                    // for info: eventInfo
                    }
                }
            }
        }
    // event : events
    }
}

Example 4 with EventColumnName

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName in project hadoop by apache.

the class TimelineEntityReader method readEvents.

/**
   * Read events from the entity table or the application table. The column name
   * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
   * if there is no info associated with the event.
   *
   * @param <T> Describes the type of column prefix.
   * @param entity entity to fill.
   * @param result HBase Result.
   * @param prefix column prefix.
   * @throws IOException if any problem is encountered while reading result.
   */
protected static <T> void readEvents(TimelineEntity entity, Result result, ColumnPrefix<T> prefix) throws IOException {
    Map<String, TimelineEvent> eventsMap = new HashMap<>();
    Map<EventColumnName, Object> eventsResult = prefix.readResults(result, new EventColumnNameConverter());
    for (Map.Entry<EventColumnName, Object> eventResult : eventsResult.entrySet()) {
        EventColumnName eventColumnName = eventResult.getKey();
        String key = eventColumnName.getId() + Long.toString(eventColumnName.getTimestamp());
        // Retrieve previously seen event to add to it
        TimelineEvent event = eventsMap.get(key);
        if (event == null) {
            // First time we're seeing this event, add it to the eventsMap
            event = new TimelineEvent();
            event.setId(eventColumnName.getId());
            event.setTimestamp(eventColumnName.getTimestamp());
            eventsMap.put(key, event);
        }
        if (eventColumnName.getInfoKey() != null) {
            event.addInfo(eventColumnName.getInfoKey(), eventResult.getValue());
        }
    }
    Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
    entity.addEvents(eventsSet);
}