Examples with Measurement io.cdap.cdap.api.dataset.lib.cube.Measurement used on opensource projects

Example 1 with Measurement

use of io.cdap.cdap.api.dataset.lib.cube.Measurement in project cdap by caskdata.

the class FactTableTest method testCache.

@Test
public void testCache() throws Exception {
    String tableName = "testCacheTable";
    String entityTableName = "testCacheEntityTable";
    InMemoryTableService.create(tableName);
    InMemoryTableService.create(entityTableName);
    int resolution = 5;
    InMemoryMetricsTable metricsTable = new InMemoryMetricsTable(tableName);
    FactTable table = new FactTable(metricsTable, new EntityTable(new InMemoryMetricsTable(entityTableName)), resolution, 2);
    // set the metrics collector for the table
    FactTableMetricsCollector metricsCollector = new FactTableMetricsCollector(resolution);
    table.setMetricsCollector(metricsCollector);
    // Initially the cache should be empty
    Assert.assertEquals(0, table.getFactCounterCache().size());
    // add some value with current ts
    long timestampNow = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) / resolution * resolution;
    List<DimensionValue> dims = dimValues("dim1", "dim2");
    List<Fact> metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    // Since no previous add to the metric store, these increment should still be considered as COUNTER, and the
    // cache should be updated.
    Assert.assertEquals(10, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(0, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow, value);
    }
    // Add metrics older than the current timestamp, these increment should still be considered as COUNTER, and the
    // cache should NOT be updated.
    metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow - 5, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    Assert.assertEquals(10, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(0, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow, value);
    }
    // Now insert metrics newer than the current timestamp, the increment should be considered as GAUGE, and the cache
    // should be updated
    metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow + 5, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    Assert.assertEquals(0, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(10, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow + 5, value);
    }
}

Example 2 with Measurement

use of io.cdap.cdap.api.dataset.lib.cube.Measurement in project cdap by caskdata.

the class FactTable method add.

public void add(List<Fact> facts) {
    // Simply collecting all rows/cols/values that need to be put to the underlying table.
    NavigableMap<byte[], NavigableMap<byte[], Long>> gaugesTable = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    NavigableMap<byte[], NavigableMap<byte[], Long>> incrementsTable = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    // this map is used to store metrics which was COUNTER type, but can be considered as GAUGE, which means it is
    // guaranteed to be a new row key in the underlying table.
    NavigableMap<byte[], NavigableMap<byte[], Long>> incGaugeTable = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    // this map is used to store the updated timestamp for the cache
    Map<FactCacheKey, Long> cacheUpdates = new HashMap<>();
    for (Fact fact : facts) {
        for (Measurement measurement : fact.getMeasurements()) {
            byte[] rowKey = codec.createRowKey(fact.getDimensionValues(), measurement.getName(), fact.getTimestamp());
            byte[] column = codec.createColumn(fact.getTimestamp());
            if (MeasureType.COUNTER == measurement.getType()) {
                if (factCounterCache != null) {
                    // round to the resolution timestamp
                    long tsToResolution = fact.getTimestamp() / resolution * resolution;
                    FactCacheKey cacheKey = new FactCacheKey(fact.getDimensionValues(), measurement.getName());
                    Long existingTs = factCounterCache.getIfPresent(cacheKey);
                    // cannot be considered as a gauge, and we should update the incrementsTable
                    if (existingTs == null || existingTs >= tsToResolution) {
                        inc(incrementsTable, rowKey, column, measurement.getValue());
                    // if the current ts is greater than existing ts, then we can consider this metric as a newly seen metric
                    // and perform gauge on this metric
                    } else {
                        inc(incGaugeTable, rowKey, column, measurement.getValue());
                    }
                    // should be updated
                    if (existingTs == null || existingTs < tsToResolution) {
                        cacheUpdates.compute(cacheKey, (key, oldValue) -> oldValue == null || tsToResolution > oldValue ? tsToResolution : oldValue);
                    }
                } else {
                    inc(incrementsTable, rowKey, column, measurement.getValue());
                }
            } else {
                gaugesTable.computeIfAbsent(rowKey, k -> Maps.newTreeMap(Bytes.BYTES_COMPARATOR)).put(column, measurement.getValue());
            }
        }
    }
    if (factCounterCache != null) {
        gaugesTable.putAll(incGaugeTable);
        factCounterCache.putAll(cacheUpdates);
    }
    // todo: replace with single call, to be able to optimize rpcs in underlying table
    timeSeriesTable.put(gaugesTable);
    timeSeriesTable.increment(incrementsTable);
    if (metrics != null) {
        metrics.increment(putCountMetric, gaugesTable.size());
        metrics.increment(incrementCountMetric, incrementsTable.size());
    }
}

Example 3 with Measurement

use of io.cdap.cdap.api.dataset.lib.cube.Measurement in project cdap by caskdata.

the class DefaultMetricStore method add.

@Override
public void add(Collection<? extends MetricValues> metricValues) {
    List<CubeFact> facts = Lists.newArrayListWithCapacity(metricValues.size());
    for (MetricValues metricValue : metricValues) {
        String scope = metricValue.getTags().get(Constants.Metrics.Tag.SCOPE);
        List<Measurement> metrics = Lists.newArrayList();
        // todo improve this logic?
        for (MetricValue metric : metricValue.getMetrics()) {
            String measureName = (scope == null ? "system." : scope + ".") + metric.getName();
            if (metric.getType() == MetricType.DISTRIBUTION) {
                // https://cdap.atlassian.net/browse/CDAP-18769
                continue;
            }
            MeasureType type = metric.getType() == MetricType.COUNTER ? MeasureType.COUNTER : MeasureType.GAUGE;
            metrics.add(new Measurement(measureName, type, metric.getValue()));
        }
        CubeFact fact = new CubeFact(metricValue.getTimestamp()).addDimensionValues(metricValue.getTags()).addMeasurements(metrics);
        facts.add(fact);
    }
    cube.get().add(facts);
}

Example 4 with Measurement

use of io.cdap.cdap.api.dataset.lib.cube.Measurement in project cdap by caskdata.

the class FactTableTest method testPreSplits.

@Test
public void testPreSplits() throws Exception {
    InMemoryTableService.create("presplitEntityTable");
    InMemoryTableService.create("presplitDataTable");
    int resolution = 10;
    int rollTimebaseInterval = 2;
    InMemoryMetricsTable metricsTable = new InMemoryMetricsTable("presplitDataTable");
    FactTable table = new FactTable(metricsTable, new EntityTable(new InMemoryMetricsTable("presplitEntityTable")), resolution, rollTimebaseInterval);
    byte[][] splits = FactTable.getSplits(3);
    long ts = System.currentTimeMillis() / 1000;
    DimensionValue dimVal1 = new DimensionValue("dim1", "value1");
    DimensionValue dimVal2 = new DimensionValue("dim2", "value2");
    DimensionValue dimVal3 = new DimensionValue("dim3", "value3");
    // first agg view: dim1
    table.add(ImmutableList.of(new Fact(ts, ImmutableList.of(dimVal1), new Measurement("metric1", MeasureType.COUNTER, 1))));
    // second agg view: dim1 & dim2
    table.add(ImmutableList.of(new Fact(ts, ImmutableList.of(dimVal1, dimVal2), new Measurement("metric1", MeasureType.COUNTER, 1))));
    // third agg view: dim3
    table.add(ImmutableList.of(new Fact(ts, ImmutableList.of(dimVal3), new Measurement("metric1", MeasureType.COUNTER, 1))));
    // Verify all written records are spread across splits
    Scanner scanner = metricsTable.scan(null, null, null);
    Row row;
    Set<Integer> splitsWithRows = Sets.newHashSet();
    while ((row = scanner.next()) != null) {
        boolean added = false;
        for (int i = 0; i < splits.length; i++) {
            if (Bytes.compareTo(row.getRow(), splits[i]) < 0) {
                splitsWithRows.add(i);
                added = true;
                break;
            }
        }
        if (!added) {
            // falls into last split
            splitsWithRows.add(splits.length);
        }
    }
    Assert.assertEquals(3, splitsWithRows.size());
}

Example 5 with Measurement

use of io.cdap.cdap.api.dataset.lib.cube.Measurement in project cdap by caskdata.

the class FactTableTest method testBasics.

@Test
public void testBasics() throws Exception {
    InMemoryTableService.create("EntityTable");
    InMemoryTableService.create("DataTable");
    int resolution = 10;
    int rollTimebaseInterval = 2;
    FactTable table = new FactTable(new InMemoryMetricsTable("DataTable"), new EntityTable(new InMemoryMetricsTable("EntityTable")), resolution, rollTimebaseInterval);
    // aligned to start of resolution bucket
    // "/1000" because time is expected to be in seconds
    long ts = ((System.currentTimeMillis() / 1000) / resolution) * resolution;
    // testing encoding with multiple dims
    List<DimensionValue> dimensionValues = ImmutableList.of(new DimensionValue("dim1", "value1"), new DimensionValue("dim2", "value2"), new DimensionValue("dim3", "value3"));
    // trying adding one by one, in same (first) time resolution bucket
    for (int i = 0; i < 5; i++) {
        for (int k = 1; k < 4; k++) {
            // note: "+i" here and below doesn't affect results, just to confirm
            // that data points are rounded to the resolution
            table.add(ImmutableList.of(new Fact(ts + i, dimensionValues, new Measurement("metric" + k, MeasureType.COUNTER, k))));
        }
    }
    // trying adding one by one, in different time resolution buckets
    for (int i = 0; i < 3; i++) {
        for (int k = 1; k < 4; k++) {
            table.add(ImmutableList.of(new Fact(ts + resolution * i + i, dimensionValues, new Measurement("metric" + k, MeasureType.COUNTER, 2 * k))));
        }
    }
    // trying adding as list
    // first incs in same (second) time resolution bucket
    List<Fact> aggs = Lists.newArrayList();
    for (int i = 0; i < 7; i++) {
        for (int k = 1; k < 4; k++) {
            aggs.add(new Fact(ts + resolution, dimensionValues, new Measurement("metric" + k, MeasureType.COUNTER, 3 * k)));
        }
    }
    // then incs in different time resolution buckets
    for (int i = 0; i < 3; i++) {
        for (int k = 1; k < 4; k++) {
            aggs.add(new Fact(ts + resolution * i, dimensionValues, new Measurement("metric" + k, MeasureType.COUNTER, 4 * k)));
        }
    }
    table.add(aggs);
    // verify each metric
    for (int k = 1; k < 4; k++) {
        FactScan scan = new FactScan(ts - 2 * resolution, ts + 3 * resolution, "metric" + k, dimensionValues);
        Table<String, List<DimensionValue>, List<TimeValue>> expected = HashBasedTable.create();
        expected.put("metric" + k, dimensionValues, ImmutableList.of(new TimeValue(ts, 11 * k), new TimeValue(ts + resolution, 27 * k), new TimeValue(ts + 2 * resolution, 6 * k)));
        assertScan(table, expected, scan);
    }
    // verify each metric within a single timeBase
    for (int k = 1; k < 4; k++) {
        FactScan scan = new FactScan(ts, ts + resolution - 1, "metric" + k, dimensionValues);
        Table<String, List<DimensionValue>, List<TimeValue>> expected = HashBasedTable.create();
        expected.put("metric" + k, dimensionValues, ImmutableList.of(new TimeValue(ts, 11 * k)));
        assertScan(table, expected, scan);
    }
    // verify all metrics with fuzzy metric in scan
    Table<String, List<DimensionValue>, List<TimeValue>> expected = HashBasedTable.create();
    for (int k = 1; k < 4; k++) {
        expected.put("metric" + k, dimensionValues, ImmutableList.of(new TimeValue(ts, 11 * k), new TimeValue(ts + resolution, 27 * k), new TimeValue(ts + 2 * resolution, 6 * k)));
    }
    // metric = null means "all"
    FactScan scan = new FactScan(ts - 2 * resolution, ts + 3 * resolution, dimensionValues);
    assertScan(table, expected, scan);
    // delete metric test
    expected.clear();
    // delete the metrics data at (timestamp + 20) resolution
    scan = new FactScan(ts + resolution * 2, ts + resolution * 3, dimensionValues);
    table.delete(scan);
    for (int k = 1; k < 4; k++) {
        expected.put("metric" + k, dimensionValues, ImmutableList.of(new TimeValue(ts, 11 * k), new TimeValue(ts + resolution, 27 * k)));
    }
    // verify deletion
    scan = new FactScan(ts - 2 * resolution, ts + 3 * resolution, dimensionValues);
    assertScan(table, expected, scan);
    // delete metrics for "metric1" at ts0 and verify deletion
    scan = new FactScan(ts, ts + 1, "metric1", dimensionValues);
    table.delete(scan);
    expected.clear();
    expected.put("metric1", dimensionValues, ImmutableList.of(new TimeValue(ts + resolution, 27)));
    scan = new FactScan(ts - 2 * resolution, ts + 3 * resolution, "metric1", dimensionValues);
    assertScan(table, expected, scan);
    // verify the next dims search
    Collection<DimensionValue> nextTags = table.findSingleDimensionValue(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableMap.of("dim1", "value1"), ts, ts + 1);
    Assert.assertEquals(ImmutableSet.of(new DimensionValue("dim2", "value2")), nextTags);
    Map<String, String> slice = Maps.newHashMap();
    slice.put("dim1", null);
    nextTags = table.findSingleDimensionValue(ImmutableList.of("dim1", "dim2", "dim3"), slice, ts, ts + 1);
    Assert.assertEquals(ImmutableSet.of(new DimensionValue("dim2", "value2")), nextTags);
    nextTags = table.findSingleDimensionValue(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableMap.of("dim1", "value1", "dim2", "value2"), ts, ts + 3);
    Assert.assertEquals(ImmutableSet.of(new DimensionValue("dim3", "value3")), nextTags);
    // add new dim values
    dimensionValues = ImmutableList.of(new DimensionValue("dim1", "value1"), new DimensionValue("dim2", "value5"), new DimensionValue("dim3", null));
    table.add(ImmutableList.of(new Fact(ts, dimensionValues, new Measurement("metric", MeasureType.COUNTER, 10))));
    dimensionValues = ImmutableList.of(new DimensionValue("dim1", "value1"), new DimensionValue("dim2", null), new DimensionValue("dim3", "value3"));
    table.add(ImmutableList.of(new Fact(ts, dimensionValues, new Measurement("metric", MeasureType.COUNTER, 10))));
    nextTags = table.findSingleDimensionValue(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableMap.of("dim1", "value1"), ts, ts + 1);
    Assert.assertEquals(ImmutableSet.of(new DimensionValue("dim2", "value2"), new DimensionValue("dim2", "value5"), new DimensionValue("dim3", "value3")), nextTags);
    // search for metric names given dims list and verify
    Collection<String> metricNames = table.findMeasureNames(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableMap.of("dim1", "value1", "dim2", "value2", "dim3", "value3"), ts, ts + 1);
    Assert.assertEquals(ImmutableSet.of("metric2", "metric3"), metricNames);
    metricNames = table.findMeasureNames(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableMap.of("dim1", "value1"), ts, ts + 1);
    Assert.assertEquals(ImmutableSet.of("metric", "metric2", "metric3"), metricNames);
    metricNames = table.findMeasureNames(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableMap.of("dim2", "value2"), ts, ts + 1);
    Assert.assertEquals(ImmutableSet.of("metric2", "metric3"), metricNames);
    metricNames = table.findMeasureNames(ImmutableList.of("dim1", "dim2", "dim3"), slice, ts, ts + 1);
    Assert.assertEquals(ImmutableSet.of("metric", "metric2", "metric3"), metricNames);
}