Examples with TimelineMetric org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric used on opensource projects

Example 21 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class HBaseTimelineWriterImpl method storeFlowMetrics.

private void storeFlowMetrics(byte[] rowKey, Set<TimelineMetric> metrics, Attribute... attributes) throws IOException {
    for (TimelineMetric metric : metrics) {
        byte[] metricColumnQualifier = stringKeyConverter.encode(metric.getId());
        Map<Long, Number> timeseries = metric.getValues();
        for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
            Long timestamp = timeseriesEntry.getKey();
            FlowRunColumnPrefix.METRIC.store(rowKey, flowRunTable, metricColumnQualifier, timestamp, timeseriesEntry.getValue(), attributes);
        }
    }
}

Example 22 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TestHBaseStorageFlowRun method testWriteFlowRunMetricsPrefix.

@Test
public void testWriteFlowRunMetricsPrefix() throws Exception {
    String cluster = "testWriteFlowRunMetricsPrefix_cluster1";
    String user = "testWriteFlowRunMetricsPrefix_user1";
    String flow = "testWriteFlowRunMetricsPrefix_flow_name";
    String flowVersion = "CF7022C10F1354";
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(System.currentTimeMillis());
    te.addEntity(entityApp1);
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        String appName = "application_11111111111111_1111";
        hbi.write(cluster, user, flow, flowVersion, 1002345678919L, appName, te);
        // write another application with same metric to this flow
        te = new TimelineEntities();
        TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(System.currentTimeMillis());
        te.addEntity(entityApp2);
        appName = "application_11111111111111_2222";
        hbi.write(cluster, user, flow, flowVersion, 1002345678918L, appName, te);
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        TimelineFilterList metricsToRetrieve = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, METRIC1.substring(0, METRIC1.indexOf("_") + 1)));
        TimelineEntity entity = hbr.getEntity(new TimelineReaderContext(cluster, user, flow, 1002345678919L, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
        assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
        Set<TimelineMetric> metrics = entity.getMetrics();
        assertEquals(1, metrics.size());
        for (TimelineMetric metric : metrics) {
            String id = metric.getId();
            Map<Long, Number> values = metric.getValues();
            assertEquals(1, values.size());
            Number value = null;
            for (Number n : values.values()) {
                value = n;
            }
            switch(id) {
                case METRIC1:
                    assertEquals(40L, value);
                    break;
                default:
                    fail("unrecognized metric: " + id);
            }
        }
        Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
        assertEquals(2, entities.size());
        int metricCnt = 0;
        for (TimelineEntity timelineEntity : entities) {
            metricCnt += timelineEntity.getMetrics().size();
        }
        assertEquals(2, metricCnt);
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 23 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TimelineEntityReader method readMetrics.

/**
   * Helper method for reading and deserializing {@link TimelineMetric} objects
   * using the specified column prefix. The timeline metrics then are added to
   * the given timeline entity.
   *
   * @param entity {@link TimelineEntity} object.
   * @param result {@link Result} object retrieved from backend.
   * @param columnPrefix Metric column prefix
   * @throws IOException if any exception is encountered while reading metrics.
   */
protected void readMetrics(TimelineEntity entity, Result result, ColumnPrefix<?> columnPrefix) throws IOException {
    NavigableMap<String, NavigableMap<Long, Number>> metricsResult = columnPrefix.readResultsWithTimestamps(result, stringKeyConverter);
    for (Map.Entry<String, NavigableMap<Long, Number>> metricResult : metricsResult.entrySet()) {
        TimelineMetric metric = new TimelineMetric();
        metric.setId(metricResult.getKey());
        // Simply assume that if the value set contains more than 1 elements, the
        // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
        TimelineMetric.Type metricType = metricResult.getValue().size() > 1 ? TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE;
        metric.setType(metricType);
        metric.addValues(metricResult.getValue());
        entity.addMetric(metric);
    }
}

Example 24 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class FileSystemTimelineReaderImpl method mergeEntities.

private static void mergeEntities(TimelineEntity entity1, TimelineEntity entity2) {
    // Ideally created time wont change except in the case of issue from client.
    if (entity2.getCreatedTime() != null && entity2.getCreatedTime() > 0) {
        entity1.setCreatedTime(entity2.getCreatedTime());
    }
    for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) {
        entity1.addConfig(configEntry.getKey(), configEntry.getValue());
    }
    for (Entry<String, Object> infoEntry : entity2.getInfo().entrySet()) {
        entity1.addInfo(infoEntry.getKey(), infoEntry.getValue());
    }
    for (Entry<String, Set<String>> isRelatedToEntry : entity2.getIsRelatedToEntities().entrySet()) {
        String type = isRelatedToEntry.getKey();
        for (String entityId : isRelatedToEntry.getValue()) {
            entity1.addIsRelatedToEntity(type, entityId);
        }
    }
    for (Entry<String, Set<String>> relatesToEntry : entity2.getRelatesToEntities().entrySet()) {
        String type = relatesToEntry.getKey();
        for (String entityId : relatesToEntry.getValue()) {
            entity1.addRelatesToEntity(type, entityId);
        }
    }
    for (TimelineEvent event : entity2.getEvents()) {
        entity1.addEvent(event);
    }
    for (TimelineMetric metric2 : entity2.getMetrics()) {
        boolean found = false;
        for (TimelineMetric metric1 : entity1.getMetrics()) {
            if (metric1.getId().equals(metric2.getId())) {
                metric1.addValues(metric2.getValues());
                found = true;
                break;
            }
        }
        if (!found) {
            entity1.addMetric(metric2);
        }
    }
}

Example 25 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TimelineStorageUtils method matchCompareFilter.

/**
   * Matches compare filter. Used for metric filters.
   *
   * @param entity entity which holds the metrics which we will match against.
   * @param compareFilter compare filter.
   * @param entityFiltersType type of filters we are trying to match.
   * @return true, if filter matches, false otherwise.
   * @throws IOException if metric filters holds non integral values.
   */
private static boolean matchCompareFilter(TimelineEntity entity, TimelineCompareFilter compareFilter, TimelineEntityFiltersType entityFiltersType) throws IOException {
    // Currently exists filter is only supported for metric filters.
    if (entityFiltersType != TimelineEntityFiltersType.METRIC) {
        return false;
    }
    // We expect only integral values(short/int/long) for metric filters.
    if (!isIntegralValue(compareFilter.getValue())) {
        throw new IOException("Metric filters has non integral values");
    }
    Map<String, TimelineMetric> metricMap = new HashMap<String, TimelineMetric>();
    for (TimelineMetric metric : entity.getMetrics()) {
        metricMap.put(metric.getId(), metric);
    }
    TimelineMetric metric = metricMap.get(compareFilter.getKey());
    if (metric == null) {
        return false;
    }
    // We will be using the latest value of metric to compare.
    return compareValues(compareFilter.getCompareOp(), metric.getValuesJAXB().firstEntry().getValue().longValue(), ((Number) compareFilter.getValue()).longValue());
}