Examples with CubeFact io.cdap.cdap.api.dataset.lib.cube.CubeFact used on opensource projects

Example 1 with CubeFact

use of io.cdap.cdap.api.dataset.lib.cube.CubeFact in project cdap by caskdata.

the class DefaultMetricStore method add.

@Override
public void add(Collection<? extends MetricValues> metricValues) {
    List<CubeFact> facts = Lists.newArrayListWithCapacity(metricValues.size());
    for (MetricValues metricValue : metricValues) {
        String scope = metricValue.getTags().get(Constants.Metrics.Tag.SCOPE);
        List<Measurement> metrics = Lists.newArrayList();
        // todo improve this logic?
        for (MetricValue metric : metricValue.getMetrics()) {
            String measureName = (scope == null ? "system." : scope + ".") + metric.getName();
            if (metric.getType() == MetricType.DISTRIBUTION) {
                // https://cdap.atlassian.net/browse/CDAP-18769
                continue;
            }
            MeasureType type = metric.getType() == MetricType.COUNTER ? MeasureType.COUNTER : MeasureType.GAUGE;
            metrics.add(new Measurement(measureName, type, metric.getValue()));
        }
        CubeFact fact = new CubeFact(metricValue.getTimestamp()).addDimensionValues(metricValue.getTags()).addMeasurements(metrics);
        facts.add(fact);
    }
    cube.get().add(facts);
}

Example 2 with CubeFact

use of io.cdap.cdap.api.dataset.lib.cube.CubeFact in project cdap by caskdata.

the class TestAppWithCube method testApp.

@Category(SlowTests.class)
@Test
public void testApp() throws Exception {
    // Deploy the application
    ApplicationManager appManager = deployApplication(AppWithCube.class);
    ServiceManager serviceManager = appManager.getServiceManager(AppWithCube.SERVICE_NAME).start();
    try {
        serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
        URL url = serviceManager.getServiceURL();
        long tsInSec = System.currentTimeMillis() / 1000;
        // round to a minute for testing minute resolution
        tsInSec = (tsInSec / 60) * 60;
        // add couple facts
        add(url, ImmutableList.of(new CubeFact(tsInSec).addDimensionValue("user", "alex").addDimensionValue("action", "click").addMeasurement("count", MeasureType.COUNTER, 1)));
        add(url, ImmutableList.of(new CubeFact(tsInSec).addDimensionValue("user", "alex").addDimensionValue("action", "click").addMeasurement("count", MeasureType.COUNTER, 1), new CubeFact(tsInSec + 1).addDimensionValue("user", "alex").addDimensionValue("action", "back").addMeasurement("count", MeasureType.COUNTER, 1), new CubeFact(tsInSec + 2).addDimensionValue("user", "alex").addDimensionValue("action", "click").addMeasurement("count", MeasureType.COUNTER, 1)));
        // search for tags
        Collection<DimensionValue> tags = searchDimensionValue(url, new CubeExploreQuery(tsInSec - 60, tsInSec + 60, 1, 100, new ArrayList<DimensionValue>()));
        Assert.assertEquals(1, tags.size());
        DimensionValue tv = tags.iterator().next();
        Assert.assertEquals("user", tv.getName());
        Assert.assertEquals("alex", tv.getValue());
        tags = searchDimensionValue(url, CubeExploreQuery.builder().from().resolution(1, TimeUnit.SECONDS).where().dimension("user", "alex").timeRange(tsInSec - 60, tsInSec + 60).limit(100).build());
        Assert.assertEquals(2, tags.size());
        Iterator<DimensionValue> iterator = tags.iterator();
        tv = iterator.next();
        Assert.assertEquals("action", tv.getName());
        Assert.assertEquals("back", tv.getValue());
        tv = iterator.next();
        Assert.assertEquals("action", tv.getName());
        Assert.assertEquals("click", tv.getValue());
        // search for measures
        Collection<String> measures = searchMeasure(url, new CubeExploreQuery(tsInSec - 60, tsInSec + 60, 1, 100, ImmutableList.of(new DimensionValue("user", "alex"))));
        Assert.assertEquals(1, measures.size());
        String measure = measures.iterator().next();
        Assert.assertEquals("count", measure);
        // query for data
        // 1-sec resolution
        Collection<TimeSeries> data = query(url, CubeQuery.builder().select().measurement("count", AggregationFunction.SUM).from(null).resolution(1, TimeUnit.SECONDS).where().dimension("action", "click").timeRange(tsInSec - 60, tsInSec + 60).limit(100).build());
        Assert.assertEquals(1, data.size());
        TimeSeries series = data.iterator().next();
        List<TimeValue> timeValues = series.getTimeValues();
        Assert.assertEquals(2, timeValues.size());
        TimeValue timeValue = timeValues.get(0);
        Assert.assertEquals(tsInSec, timeValue.getTimestamp());
        Assert.assertEquals(2, timeValue.getValue());
        timeValue = timeValues.get(1);
        Assert.assertEquals(tsInSec + 2, timeValue.getTimestamp());
        Assert.assertEquals(1, timeValue.getValue());
        // 60-sec resolution
        data = query(url, new CubeQuery(null, tsInSec - 60, tsInSec + 60, 60, 100, ImmutableMap.of("count", AggregationFunction.SUM), ImmutableMap.of("action", "click"), new ArrayList<String>(), null, null));
        Assert.assertEquals(1, data.size());
        series = data.iterator().next();
        timeValues = series.getTimeValues();
        Assert.assertEquals(1, timeValues.size());
        timeValue = timeValues.get(0);
        Assert.assertEquals(tsInSec, timeValue.getTimestamp());
        Assert.assertEquals(3, timeValue.getValue());
    } finally {
        serviceManager.stop();
        serviceManager.waitForStopped(10, TimeUnit.SECONDS);
    }
}

Example 3 with CubeFact

use of io.cdap.cdap.api.dataset.lib.cube.CubeFact in project cdap by caskdata.

the class DefaultCube method add.

@Override
public void add(Collection<? extends CubeFact> facts) {
    List<Fact> toWrite = Lists.newArrayList();
    int dimValuesCount = 0;
    for (CubeFact fact : facts) {
        for (Map.Entry<String, ? extends Aggregation> aggEntry : aggregations.entrySet()) {
            Aggregation agg = aggEntry.getValue();
            AggregationAlias aggregationAlias = null;
            if (aggregationAliasMap.containsKey(aggEntry.getKey())) {
                aggregationAlias = aggregationAliasMap.get(aggEntry.getKey());
            }
            if (agg.accept(fact)) {
                List<DimensionValue> dimensionValues = Lists.newArrayList();
                for (String dimensionName : agg.getDimensionNames()) {
                    String dimensionValueKey = aggregationAlias == null ? dimensionName : aggregationAlias.getAlias(dimensionName);
                    dimensionValues.add(new DimensionValue(dimensionName, fact.getDimensionValues().get(dimensionValueKey)));
                    dimValuesCount++;
                }
                toWrite.add(new Fact(fact.getTimestamp(), dimensionValues, fact.getMeasurements()));
            }
        }
    }
    Map<Integer, Future<?>> futures = new HashMap<>();
    for (Map.Entry<Integer, FactTable> table : resolutionToFactTable.entrySet()) {
        futures.put(table.getKey(), executorService.submit(() -> table.getValue().add(toWrite)));
    }
    boolean failed = false;
    Exception failedException = null;
    StringBuilder failedMessage = new StringBuilder("Failed to add metrics to ");
    for (Map.Entry<Integer, Future<?>> future : futures.entrySet()) {
        try {
            Uninterruptibles.getUninterruptibly(future.getValue());
        } catch (ExecutionException e) {
            if (!failed) {
                failed = true;
                failedMessage.append(String.format("the %d resolution table", future.getKey()));
            } else {
                failedMessage.append(String.format(", the %d resolution table", future.getKey()));
            }
            if (failedException == null) {
                failedException = e;
            } else {
                failedException.addSuppressed(e);
            }
        }
    }
    if (failed) {
        throw new RuntimeException(failedMessage.append(".").toString(), failedException);
    }
    incrementMetric("cube.cubeFact.add.request.count", 1);
    incrementMetric("cube.cubeFact.added.count", facts.size());
    incrementMetric("cube.tsFact.created.count", toWrite.size());
    incrementMetric("cube.tsFact.created.dimValues.count", dimValuesCount);
    incrementMetric("cube.tsFact.added.count", toWrite.size() * resolutionToFactTable.size());
}