Examples with TimeValue io.cdap.cdap.api.dataset.lib.cube.TimeValue used on opensource projects

Example 6 with TimeValue

use of io.cdap.cdap.api.dataset.lib.cube.TimeValue in project cdap by caskdata.

the class FactTableTest method testMaxResolution.

@Test
public void testMaxResolution() throws Exception {
    // we use Integer.MAX_VALUE as resolution to compute all-time total values
    InMemoryTableService.create("TotalsEntityTable");
    InMemoryTableService.create("TotalsDataTable");
    int resolution = Integer.MAX_VALUE;
    // should not matter when resolution is max
    int rollTimebaseInterval = 3600;
    FactTable table = new FactTable(new InMemoryMetricsTable("TotalsDataTable"), new EntityTable(new InMemoryMetricsTable("TotalsEntityTable")), resolution, rollTimebaseInterval);
    // ts is expected in seconds
    long ts = System.currentTimeMillis() / 1000;
    int count = 1000;
    for (int i = 0; i < count; i++) {
        for (int k = 0; k < 10; k++) {
            // shift one day
            writeInc(table, "metric" + k, ts + i * 60 * 60 * 24, i * k, "dim" + k, "value" + k);
        }
    }
    for (int k = 0; k < 10; k++) {
        // 0, 0 should match timestamp of all data points
        FactScan scan = new FactScan(0, 0, "metric" + k, dimValues("dim" + k, "value" + k));
        Table<String, List<DimensionValue>, List<TimeValue>> expected = HashBasedTable.create();
        expected.put("metric" + k, dimValues("dim" + k, "value" + k), ImmutableList.of(new TimeValue(0, k * count * (count - 1) / 2)));
        assertScan(table, expected, scan);
    }
}

Example 7 with TimeValue

use of io.cdap.cdap.api.dataset.lib.cube.TimeValue in project cdap by caskdata.

the class SupportBundleRuntimeInfoTask method queryMetrics.

public JsonObject queryMetrics(String runId, String configuration, long startTs, long stopTs) {
    // startTs from run time but metrics already starts before that time
    int startQueryTime = (int) (startTs - 5000);
    JsonObject metrics = new JsonObject();
    try {
        // This program type tag can be null
        String typeTag = getMetricsTag(programType);
        Map<String, String> queryTags = new HashMap<>();
        queryTags.put("namespace", namespaceId.getNamespace());
        queryTags.put("app", appId.getApplication());
        queryTags.put("run", runId);
        if (typeTag != null) {
            queryTags.put(typeTag, programName.getProgram());
        }
        Collection<String> metricsNameList = remoteMetricsSystemClient.search(queryTags);
        List<MetricTimeSeries> metricTimeSeriesList = new ArrayList<>(remoteMetricsSystemClient.query(startQueryTime, (int) (stopTs), queryTags, metricsNameList));
        for (MetricTimeSeries timeSeries : metricTimeSeriesList) {
            if (!metrics.has(timeSeries.getMetricName())) {
                metrics.add(timeSeries.getMetricName(), new JsonArray());
            }
            for (TimeValue timeValue : timeSeries.getTimeValues()) {
                JsonObject time = new JsonObject();
                time.addProperty("time", timeValue.getTimestamp());
                time.addProperty("value", timeValue.getValue());
                metrics.getAsJsonArray(timeSeries.getMetricName()).add(time);
            }
        }
    } catch (IOException e) {
        LOG.warn("Failed to find metrics with run {} ", runId, e);
        return null;
    }
    return metrics;
}

Example 8 with TimeValue

use of io.cdap.cdap.api.dataset.lib.cube.TimeValue in project cdap by caskdata.

the class RemoteMetricsSystemClient method query.

@Override
public Collection<MetricTimeSeries> query(int start, int end, int resolution, Map<String, String> tags, Collection<String> metrics, Collection<String> groupByTags) throws IOException {
    String metricsParam = Joiner.on("&metric=").join(metrics);
    if (!metricsParam.isEmpty()) {
        metricsParam = "&metric=" + metricsParam;
    }
    String tagsParam = Joiner.on("&tag=").withKeyValueSeparator(":").join(tags);
    if (!tagsParam.isEmpty()) {
        tagsParam = "&tag=" + tagsParam;
    }
    String groupByParam = Joiner.on("&groupBy=").join(groupByTags);
    if (!groupByParam.isEmpty()) {
        groupByParam = "&groupBy=" + groupByParam;
    }
    // Only query for aggregate metrics. Currently that's the only use case.
    String queryString = "aggregate=true&start=" + start + "&end=" + end + "&resolution=" + resolution + "s" + metricsParam + tagsParam + groupByParam;
    URL url = getBaseURI().resolve("query?" + queryString).toURL();
    HttpResponse response = HttpRequests.execute(HttpRequest.post(url).build(), REQUEST_CONFIG);
    if (response.getResponseCode() != HttpURLConnection.HTTP_OK) {
        throw new IOException("Failed to query for metrics " + metrics + ", with tags " + tags + ". Error code " + response.getResponseCode() + ", message " + response.getResponseBodyAsString());
    }
    MetricQueryResult queryResult = GSON.fromJson(response.getResponseBodyAsString(), MetricQueryResult.class);
    List<MetricTimeSeries> result = new ArrayList<>();
    for (MetricQueryResult.TimeSeries timeSeries : queryResult.getSeries()) {
        List<TimeValue> timeValues = Arrays.stream(timeSeries.getData()).map(tv -> new TimeValue(tv.getTime(), tv.getValue())).collect(Collectors.toList());
        result.add(new MetricTimeSeries(timeSeries.getMetricName(), timeSeries.getGrouping(), timeValues));
    }
    return result;
}

Example 9 with TimeValue

use of io.cdap.cdap.api.dataset.lib.cube.TimeValue in project cdap by caskdata.

the class MetricsQueryHelper method decorate.

private MetricQueryResult.TimeValue[] decorate(List<TimeValue> points) {
    MetricQueryResult.TimeValue[] timeValues = new MetricQueryResult.TimeValue[points.size()];
    int k = 0;
    for (TimeValue timeValue : points) {
        timeValues[k++] = new MetricQueryResult.TimeValue(timeValue.getTimestamp(), timeValue.getValue());
    }
    return timeValues;
}

Example 10 with TimeValue

use of io.cdap.cdap.api.dataset.lib.cube.TimeValue in project cdap by caskdata.

the class TestAppWithCube method testApp.

@Category(SlowTests.class)
@Test
public void testApp() throws Exception {
    // Deploy the application
    ApplicationManager appManager = deployApplication(AppWithCube.class);
    ServiceManager serviceManager = appManager.getServiceManager(AppWithCube.SERVICE_NAME).start();
    try {
        serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
        URL url = serviceManager.getServiceURL();
        long tsInSec = System.currentTimeMillis() / 1000;
        // round to a minute for testing minute resolution
        tsInSec = (tsInSec / 60) * 60;
        // add couple facts
        add(url, ImmutableList.of(new CubeFact(tsInSec).addDimensionValue("user", "alex").addDimensionValue("action", "click").addMeasurement("count", MeasureType.COUNTER, 1)));
        add(url, ImmutableList.of(new CubeFact(tsInSec).addDimensionValue("user", "alex").addDimensionValue("action", "click").addMeasurement("count", MeasureType.COUNTER, 1), new CubeFact(tsInSec + 1).addDimensionValue("user", "alex").addDimensionValue("action", "back").addMeasurement("count", MeasureType.COUNTER, 1), new CubeFact(tsInSec + 2).addDimensionValue("user", "alex").addDimensionValue("action", "click").addMeasurement("count", MeasureType.COUNTER, 1)));
        // search for tags
        Collection<DimensionValue> tags = searchDimensionValue(url, new CubeExploreQuery(tsInSec - 60, tsInSec + 60, 1, 100, new ArrayList<DimensionValue>()));
        Assert.assertEquals(1, tags.size());
        DimensionValue tv = tags.iterator().next();
        Assert.assertEquals("user", tv.getName());
        Assert.assertEquals("alex", tv.getValue());
        tags = searchDimensionValue(url, CubeExploreQuery.builder().from().resolution(1, TimeUnit.SECONDS).where().dimension("user", "alex").timeRange(tsInSec - 60, tsInSec + 60).limit(100).build());
        Assert.assertEquals(2, tags.size());
        Iterator<DimensionValue> iterator = tags.iterator();
        tv = iterator.next();
        Assert.assertEquals("action", tv.getName());
        Assert.assertEquals("back", tv.getValue());
        tv = iterator.next();
        Assert.assertEquals("action", tv.getName());
        Assert.assertEquals("click", tv.getValue());
        // search for measures
        Collection<String> measures = searchMeasure(url, new CubeExploreQuery(tsInSec - 60, tsInSec + 60, 1, 100, ImmutableList.of(new DimensionValue("user", "alex"))));
        Assert.assertEquals(1, measures.size());
        String measure = measures.iterator().next();
        Assert.assertEquals("count", measure);
        // query for data
        // 1-sec resolution
        Collection<TimeSeries> data = query(url, CubeQuery.builder().select().measurement("count", AggregationFunction.SUM).from(null).resolution(1, TimeUnit.SECONDS).where().dimension("action", "click").timeRange(tsInSec - 60, tsInSec + 60).limit(100).build());
        Assert.assertEquals(1, data.size());
        TimeSeries series = data.iterator().next();
        List<TimeValue> timeValues = series.getTimeValues();
        Assert.assertEquals(2, timeValues.size());
        TimeValue timeValue = timeValues.get(0);
        Assert.assertEquals(tsInSec, timeValue.getTimestamp());
        Assert.assertEquals(2, timeValue.getValue());
        timeValue = timeValues.get(1);
        Assert.assertEquals(tsInSec + 2, timeValue.getTimestamp());
        Assert.assertEquals(1, timeValue.getValue());
        // 60-sec resolution
        data = query(url, new CubeQuery(null, tsInSec - 60, tsInSec + 60, 60, 100, ImmutableMap.of("count", AggregationFunction.SUM), ImmutableMap.of("action", "click"), new ArrayList<String>(), null, null));
        Assert.assertEquals(1, data.size());
        series = data.iterator().next();
        timeValues = series.getTimeValues();
        Assert.assertEquals(1, timeValues.size());
        timeValue = timeValues.get(0);
        Assert.assertEquals(tsInSec, timeValue.getTimestamp());
        Assert.assertEquals(3, timeValue.getValue());
    } finally {
        serviceManager.stop();
        serviceManager.waitForStopped(10, TimeUnit.SECONDS);
    }
}