Examples with Cube io.cdap.cdap.api.dataset.lib.cube.Cube used on opensource projects

Example 6 with Cube

use of io.cdap.cdap.api.dataset.lib.cube.Cube in project cdap by cdapio.

the class AbstractCubeTest method testMetricsAggregationOptionLatest.

@Test
public void testMetricsAggregationOptionLatest() throws Exception {
    Aggregation agg = new DefaultAggregation(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableList.of("dim1"));
    int resolution = 1;
    Cube cube = getCube("testAggOptionLatest", new int[] { resolution }, ImmutableMap.of("agg", agg));
    Map<String, String> aggDims = new LinkedHashMap<>();
    aggDims.put("dim1", "tag1");
    aggDims.put("dim2", "tag2");
    aggDims.put("dim3", "tag3");
    // write 100 data points to agg
    for (int i = 1; i <= 100; i++) {
        writeGauge(cube, "metric1", i, i, aggDims);
    }
    // query for latest, should have the latest value for each interval, 20, 40, 60, 80, 100
    CubeQuery query = new CubeQuery(null, 0, 200, 1, 5, ImmutableMap.of("metric1", AggregationFunction.SUM), aggDims, Collections.emptyList(), AggregationOption.LATEST, null);
    List<TimeSeries> result = new ArrayList<>(cube.query(query));
    Assert.assertEquals(1, result.size());
    List<TimeValue> timeValues = result.get(0).getTimeValues();
    for (int i = 0; i < timeValues.size(); i++) {
        Assert.assertEquals(20 * (i + 1), timeValues.get(i).getValue());
    }
}

Example 7 with Cube

use of io.cdap.cdap.api.dataset.lib.cube.Cube in project cdap by cdapio.

the class AbstractCubeTest method testMetricsAggregationOptionSum.

@Test
public void testMetricsAggregationOptionSum() throws Exception {
    // two aggregation groups
    Aggregation agg1 = new DefaultAggregation(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableList.of("dim1"));
    Aggregation agg2 = new DefaultAggregation(ImmutableList.of("dim1", "dim2"), ImmutableList.of("dim1"));
    int resolution = 1;
    Cube cube = getCube("testAggOptionSum", new int[] { resolution }, ImmutableMap.of("agg1", agg1, "agg2", agg2));
    Map<String, String> agg1Dims = new LinkedHashMap<>();
    agg1Dims.put("dim1", "tag1");
    agg1Dims.put("dim2", "tag2");
    agg1Dims.put("dim3", "tag3");
    Map<String, String> agg2Dims = new LinkedHashMap<>();
    agg2Dims.put("dim1", "tag1");
    agg2Dims.put("dim2", "tag4");
    // write 100 data points to agg1
    for (int i = 1; i <= 100; i++) {
        writeInc(cube, "metric1", i, 1, agg1Dims);
        writeInc(cube, "metric2", i, 2, agg1Dims);
    }
    // write 50 data points to agg2
    for (int i = 1; i <= 50; i++) {
        writeInc(cube, "metric1", i, 3, agg2Dims);
    }
    // test limit must be greater than 0
    CubeQuery query = new CubeQuery(null, 0, 200, 1, 0, ImmutableMap.of("metric1", AggregationFunction.SUM), agg1Dims, Collections.emptyList(), AggregationOption.SUM, null);
    try {
        cube.query(query);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    query = new CubeQuery(null, 0, 200, 1, -10, ImmutableMap.of("metric1", AggregationFunction.SUM), agg1Dims, Collections.emptyList(), AggregationOption.SUM, null);
    try {
        cube.query(query);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    // test a limit greater than the number data points, all the data points should be returned
    query = new CubeQuery(null, 0, 200, 1, 200, ImmutableMap.of("metric1", AggregationFunction.SUM, "metric2", AggregationFunction.SUM), agg1Dims, Collections.emptyList(), AggregationOption.SUM, null);
    List<TimeSeries> result = new ArrayList<>(cube.query(query));
    Assert.assertEquals(2, result.size());
    verifySumAggregation(result.get(0), "metric1", 100, 1, 1, 0, 0);
    verifySumAggregation(result.get(1), "metric2", 100, 2, 1, 0, 0);
    // test aggregation option with sum for metric1 and metric2 for agg1, 5 data points for agg1 should get returned
    // for both metric1 and metric2
    query = new CubeQuery(null, 0, 200, 1, 5, ImmutableMap.of("metric1", AggregationFunction.SUM, "metric2", AggregationFunction.SUM), agg1Dims, Collections.emptyList(), AggregationOption.SUM, null);
    result = new ArrayList<>(cube.query(query));
    Assert.assertEquals(2, result.size());
    // metric1 increment by 1 per second, so sum will be 100/5=20, metric2 increment by 2 per second, so sum will be
    // 200/5=40
    verifySumAggregation(result.get(0), "metric1", 5, 20, 20, 0, 0);
    verifySumAggregation(result.get(1), "metric2", 5, 40, 20, 0, 0);
    // test aggregation option with sum for metric1 with tag name dim1, it should return two time series for agg1 and
    // agg2 for metric1, each with 5 data points
    query = new CubeQuery(null, 0, 200, 1, 5, ImmutableMap.of("metric1", AggregationFunction.SUM), ImmutableMap.of("dim1", "tag1"), ImmutableList.of("dim2"), AggregationOption.SUM, null);
    result = new ArrayList<>(cube.query(query));
    Assert.assertEquals(2, result.size());
    // agg1 gets increment by 1 for 100 seconds, so sum will be 100/5=20, agg2 gets increment by 3 for 50 seconds, so
    // sum will be 3*50/5=30
    verifySumAggregation(result.get(0), "metric1", 5, 30, 10, 0, 0);
    verifySumAggregation(result.get(1), "metric1", 5, 20, 20, 0, 0);
    // test metric1 with count 9, this will have a remainder 100%9=1, so there will be 9 aggregated data points, each
    // with partition size 11
    query = new CubeQuery(null, 0, 200, 1, 9, ImmutableMap.of("metric1", AggregationFunction.SUM), agg1Dims, Collections.emptyList(), AggregationOption.SUM, null);
    result = new ArrayList<>(cube.query(query));
    Assert.assertEquals(1, result.size());
    // the rest data points have sum 11
    verifySumAggregation(result.get(0), "metric1", 9, 11, 11, 1, 1);
    // test metric1 with count 70, this will have a remainder 100%70=30, so the result will have last 70 data points,
    // the first 30 data points will be ignored
    query = new CubeQuery(null, 0, 200, 1, 70, ImmutableMap.of("metric1", AggregationFunction.SUM), agg1Dims, Collections.emptyList(), AggregationOption.SUM, null);
    result = new ArrayList<>(cube.query(query));
    Assert.assertEquals(1, result.size());
    // the rest data points have sum 11
    verifySumAggregation(result.get(0), "metric1", 70, 1, 1, 30, 30);
}

Example 8 with Cube

use of io.cdap.cdap.api.dataset.lib.cube.Cube in project cdap by cdapio.

the class CubeDatasetTest method getCubeInternal.

private Cube getCubeInternal(String name, int[] resolutions, Map<String, ? extends Aggregation> aggregations) throws Exception {
    DatasetProperties props = configureProperties(resolutions, aggregations);
    DatasetId id = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset(name);
    if (dsFrameworkUtil.getInstance(id) == null) {
        dsFrameworkUtil.createInstance(Cube.class.getName(), id, props);
    }
    return dsFrameworkUtil.getInstance(id);
}

Example 9 with Cube

use of io.cdap.cdap.api.dataset.lib.cube.Cube in project cdap by cdapio.

the class AbstractCubeTest method testMetricDeletion.

@Test
public void testMetricDeletion() throws Exception {
    // two aggregation groups with different orders
    Aggregation agg1 = new DefaultAggregation(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableList.of("dim1"));
    Aggregation agg2 = new DefaultAggregation(ImmutableList.of("dim1", "dim3"), ImmutableList.of("dim3"));
    int resolution = 1;
    Cube cube = getCube("testDeletion", new int[] { resolution }, ImmutableMap.of("agg1", agg1, "agg2", agg2));
    Map<String, String> agg1Dims = new LinkedHashMap<>();
    agg1Dims.put("dim1", "1");
    agg1Dims.put("dim2", "1");
    agg1Dims.put("dim3", "1");
    Map<String, String> agg2Dims = new LinkedHashMap<>();
    agg2Dims.put("dim1", "1");
    agg2Dims.put("dim3", "1");
    // write some data
    writeInc(cube, "metric1", 1, 1, agg1Dims);
    writeInc(cube, "metric2", 3, 3, agg2Dims);
    // verify data is there
    verifyCountQuery(cube, 0, 15, resolution, "metric1", AggregationFunction.SUM, agg1Dims, ImmutableList.of(), ImmutableList.of(new TimeSeries("metric1", new HashMap<>(), timeValues(1, 1))));
    verifyCountQuery(cube, 0, 15, resolution, "metric2", AggregationFunction.SUM, agg2Dims, ImmutableList.of(), ImmutableList.of(new TimeSeries("metric2", new HashMap<>(), timeValues(3, 3))));
    // delete metrics from agg2
    Predicate<List<String>> predicate = aggregates -> Collections.indexOfSubList(aggregates, new ArrayList<>(agg2Dims.keySet())) == 0;
    CubeDeleteQuery query = new CubeDeleteQuery(0, 15, resolution, agg2Dims, Collections.emptySet(), predicate);
    cube.delete(query);
    // agg1 data should still be there
    verifyCountQuery(cube, 0, 15, resolution, "metric1", AggregationFunction.SUM, agg1Dims, ImmutableList.of(), ImmutableList.of(new TimeSeries("metric1", new HashMap<>(), timeValues(1, 1))));
    // agg2 data should get deleted
    verifyCountQuery(cube, 0, 15, resolution, "metric2", AggregationFunction.SUM, agg2Dims, ImmutableList.of(), ImmutableList.of());
    // delete metrics remain for agg1
    predicate = aggregates -> Collections.indexOfSubList(aggregates, new ArrayList<>(agg1Dims.keySet())) == 0;
    query = new CubeDeleteQuery(0, 15, resolution, agg1Dims, Collections.emptySet(), predicate);
    cube.delete(query);
    verifyCountQuery(cube, 0, 15, resolution, "metric1", AggregationFunction.SUM, agg1Dims, ImmutableList.of(), ImmutableList.of());
}

Example 10 with Cube

use of io.cdap.cdap.api.dataset.lib.cube.Cube in project cdap by cdapio.

the class AbstractCubeTest method testInterpolate.

@Test
public void testInterpolate() throws Exception {
    Aggregation agg1 = new DefaultAggregation(ImmutableList.of("dim1", "dim2", "dim3"), ImmutableList.of("dim1", "dim2", "dim3"));
    int resolution = 1;
    Cube cube = getCube("myInterpolatedCube", new int[] { resolution }, ImmutableMap.of("agg1", agg1));
    // test step interpolation
    long startTs = 1;
    long endTs = 10;
    writeInc(cube, "metric1", startTs, 5, "1", "1", "1");
    writeInc(cube, "metric1", endTs, 3, "1", "1", "1");
    List<TimeValue> expectedTimeValues = Lists.newArrayList();
    for (long i = startTs; i < endTs; i++) {
        expectedTimeValues.add(new TimeValue(i, 5));
    }
    expectedTimeValues.add(new TimeValue(endTs, 3));
    verifyCountQuery(cube, startTs, endTs, resolution, "metric1", AggregationFunction.SUM, ImmutableMap.of("dim1", "1", "dim2", "1", "dim3", "1"), new ArrayList<String>(), ImmutableList.of(new TimeSeries("metric1", new HashMap<String, String>(), expectedTimeValues)), new Interpolators.Step());
    Map<String, String> deleteTags = new LinkedHashMap<>();
    deleteTags.put("dim1", "1");
    deleteTags.put("dim2", "1");
    deleteTags.put("dim3", "1");
    Predicate<List<String>> predicate = aggregates -> Collections.indexOfSubList(aggregates, new ArrayList<>(deleteTags.keySet())) == 0;
    CubeDeleteQuery query = new CubeDeleteQuery(startTs, endTs, resolution, deleteTags, Collections.singletonList("metric1"), predicate);
    cube.delete(query);
    // test small-slope linear interpolation
    startTs = 1;
    endTs = 5;
    writeInc(cube, "metric1", startTs, 5, "1", "1", "1");
    writeInc(cube, "metric1", endTs, 3, "1", "1", "1");
    verifyCountQuery(cube, startTs, endTs, resolution, "metric1", AggregationFunction.SUM, ImmutableMap.of("dim1", "1", "dim2", "1", "dim3", "1"), new ArrayList<String>(), ImmutableList.of(new TimeSeries("metric1", new HashMap<String, String>(), timeValues(1, 5, 2, 5, 3, 4, 4, 4, 5, 3))), new Interpolators.Linear());
    query = new CubeDeleteQuery(startTs, endTs, resolution, deleteTags, Collections.singletonList("metric1"), predicate);
    cube.delete(query);
    // test big-slope linear interpolation
    writeInc(cube, "metric1", startTs, 100, "1", "1", "1");
    writeInc(cube, "metric1", endTs, 500, "1", "1", "1");
    verifyCountQuery(cube, startTs, endTs, resolution, "metric1", AggregationFunction.SUM, ImmutableMap.of("dim1", "1", "dim2", "1", "dim3", "1"), new ArrayList<String>(), ImmutableList.of(new TimeSeries("metric1", new HashMap<String, String>(), timeValues(1, 100, 2, 200, 3, 300, 4, 400, 5, 500))), new Interpolators.Linear());
    cube.delete(query);
    // test limit on Interpolate
    long limit = 20;
    writeInc(cube, "metric1", 0, 10, "1", "1", "1");
    writeInc(cube, "metric1", limit + 1, 50, "1", "1", "1");
    expectedTimeValues.clear();
    expectedTimeValues.add(new TimeValue(0, 10));
    for (long i = 1; i <= limit; i++) {
        expectedTimeValues.add(new TimeValue(i, 0));
    }
    expectedTimeValues.add(new TimeValue(limit + 1, 50));
    verifyCountQuery(cube, 0, 21, resolution, "metric1", AggregationFunction.SUM, ImmutableMap.of("dim1", "1", "dim2", "1", "dim3", "1"), new ArrayList<String>(), ImmutableList.of(new TimeSeries("metric1", new HashMap<String, String>(), expectedTimeValues)), new Interpolators.Step(limit));
}