Examples with DimensionValue io.cdap.cdap.api.dataset.lib.cube.DimensionValue used on opensource projects

Example 11 with DimensionValue

use of io.cdap.cdap.api.dataset.lib.cube.DimensionValue in project cdap by cdapio.

the class DefaultMetricStore method findNextAvailableTags.

@Override
public Collection<TagValue> findNextAvailableTags(MetricSearchQuery query) {
    Collection<DimensionValue> tags = cube.get().findDimensionValues(buildCubeSearchQuery(query));
    Collection<TagValue> result = Lists.newArrayList();
    for (DimensionValue dimensionValue : tags) {
        result.add(new TagValue(dimensionValue.getName(), dimensionValue.getValue()));
    }
    return result;
}

Example 12 with DimensionValue

use of io.cdap.cdap.api.dataset.lib.cube.DimensionValue in project cdap by cdapio.

the class FactTableTest method testCache.

@Test
public void testCache() throws Exception {
    String tableName = "testCacheTable";
    String entityTableName = "testCacheEntityTable";
    InMemoryTableService.create(tableName);
    InMemoryTableService.create(entityTableName);
    int resolution = 5;
    InMemoryMetricsTable metricsTable = new InMemoryMetricsTable(tableName);
    FactTable table = new FactTable(metricsTable, new EntityTable(new InMemoryMetricsTable(entityTableName)), resolution, 2);
    // set the metrics collector for the table
    FactTableMetricsCollector metricsCollector = new FactTableMetricsCollector(resolution);
    table.setMetricsCollector(metricsCollector);
    // Initially the cache should be empty
    Assert.assertEquals(0, table.getFactCounterCache().size());
    // add some value with current ts
    long timestampNow = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) / resolution * resolution;
    List<DimensionValue> dims = dimValues("dim1", "dim2");
    List<Fact> metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    // Since no previous add to the metric store, these increment should still be considered as COUNTER, and the
    // cache should be updated.
    Assert.assertEquals(10, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(0, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow, value);
    }
    // Add metrics older than the current timestamp, these increment should still be considered as COUNTER, and the
    // cache should NOT be updated.
    metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow - 5, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    Assert.assertEquals(10, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(0, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow, value);
    }
    // Now insert metrics newer than the current timestamp, the increment should be considered as GAUGE, and the cache
    // should be updated
    metrics = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        metrics.add(new Fact(timestampNow + 5, dims, new Measurement("metric" + i, MeasureType.COUNTER, 1)));
    }
    table.add(metrics);
    Assert.assertEquals(0, metricsCollector.getLastIncrementSize());
    Assert.assertEquals(10, metricsCollector.getLastGaugeSize());
    Assert.assertEquals(10, table.getFactCounterCache().size());
    for (long value : table.getFactCounterCache().asMap().values()) {
        Assert.assertEquals(timestampNow + 5, value);
    }
}

Example 13 with DimensionValue

use of io.cdap.cdap.api.dataset.lib.cube.DimensionValue in project cdap by cdapio.

the class FactScanner method createIterator.

private Iterator<FactScanResult> createIterator() {
    return new AbstractIterator<FactScanResult>() {

        @Override
        protected FactScanResult computeNext() {
            Row rowResult;
            while ((rowResult = scanner.next()) != null) {
                rowScanned++;
                byte[] rowKey = rowResult.getRow();
                // Decode context and metric from key
                String measureName = codec.getMeasureName(rowKey);
                // if measureNames is empty we include all metrics
                if (!measureNames.isEmpty() && !measureNames.contains(measureName)) {
                    continue;
                }
                // todo: codec.getDimensionValues(rowKey) needs to un-encode dimension names which may result in read in
                // entity table (depending on the cache and its state). To avoid that, we can pass to scanner the
                // list of dimension names as we *always* know it (it is given) at the time of scanning
                List<DimensionValue> dimensionValues = codec.getDimensionValues(rowKey);
                boolean exhausted = false;
                List<TimeValue> timeValues = Lists.newLinkedList();
                // todo: entry set is ordered by ts?
                for (Map.Entry<byte[], byte[]> columnValue : rowResult.getColumns().entrySet()) {
                    long ts = codec.getTimestamp(rowKey, columnValue.getKey());
                    if (ts < startTs) {
                        continue;
                    }
                    if (ts > endTs) {
                        exhausted = true;
                        break;
                    }
                    // todo: move Bytes.toLong into codec?
                    TimeValue timeValue = new TimeValue(ts, Bytes.toLong(columnValue.getValue()));
                    timeValues.add(timeValue);
                }
                if (timeValues.isEmpty() && exhausted) {
                    break;
                }
                // todo: can return empty list, if all data is < startTs or > endTs
                return new FactScanResult(measureName, dimensionValues, timeValues);
            }
            scanner.close();
            return endOfData();
        }
    };
}

Example 14 with DimensionValue

use of io.cdap.cdap.api.dataset.lib.cube.DimensionValue in project cdap by cdapio.

the class FactTable method findSingleDimensionValue.

/**
 * Searches for first non-null valued dimensions in records that contain given list of dimensions and match given
 * dimension values in given time range. Returned dimension values are those that are not defined in given
 * dimension values.
 * @param allDimensionNames list of all dimension names to be present in the record
 * @param dimensionSlice dimension values to filter by, {@code null} means any non-null value.
 * @param startTs start of the time range, in seconds
 * @param endTs end of the time range, in seconds
 * @return {@link Set} of {@link DimensionValue}s
 */
// todo: pass a limit on number of dimensionValues returned
// todo: kinda not cool API when we expect null values in a map...
public Set<DimensionValue> findSingleDimensionValue(List<String> allDimensionNames, Map<String, String> dimensionSlice, long startTs, long endTs) {
    // Algorithm, briefly:
    // We scan in the records which have given allDimensionNames. We use dimensionSlice as a criteria for scan.
    // If record from the scan has non-null values in the dimensions which are not specified in dimensionSlice,
    // we use first of such dimension as a value to return.
    // When we find value to return, since we only fill a single dimension, we are not interested in drilling down
    // further and instead attempt to fast-forward (jump) to a record that has different value in that dimension.
    // Thus we find all results.
    List<DimensionValue> allDimensions = Lists.newArrayList();
    List<Integer> dimToFillIndexes = Lists.newArrayList();
    for (int i = 0; i < allDimensionNames.size(); i++) {
        String dimensionName = allDimensionNames.get(i);
        if (!dimensionSlice.containsKey(dimensionName)) {
            dimToFillIndexes.add(i);
            allDimensions.add(new DimensionValue(dimensionName, null));
        } else {
            DimensionValue dimensionValue = new DimensionValue(dimensionName, dimensionSlice.get(dimensionName));
            allDimensions.add(dimensionValue);
        }
    }
    // If provided dimensions contain all values filled in, there's nothing to look for
    if (dimToFillIndexes.isEmpty()) {
        return Collections.emptySet();
    }
    Set<DimensionValue> result = Sets.newHashSet();
    int scans = 0;
    int scannedRecords = 0;
    // build a scan
    byte[] startRow = codec.createStartRowKey(allDimensions, null, startTs, false);
    byte[] endRow = codec.createEndRowKey(allDimensions, null, endTs, false);
    endRow = Bytes.stopKeyForPrefix(endRow);
    FuzzyRowFilter fuzzyRowFilter = createFuzzyRowFilter(new FactScan(startTs, endTs, Collections.emptyList(), allDimensions), startRow);
    Scanner scanner = timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter);
    scans++;
    try {
        Row rowResult;
        while ((rowResult = scanner.next()) != null) {
            scannedRecords++;
            // todo: make configurable
            if (scannedRecords > MAX_RECORDS_TO_SCAN_DURING_SEARCH) {
                break;
            }
            byte[] rowKey = rowResult.getRow();
            // filter out columns by time range (scan configuration only filters whole rows)
            if (codec.getTimestamp(rowKey, codec.createColumn(startTs)) < startTs) {
                continue;
            }
            if (codec.getTimestamp(rowKey, codec.createColumn(endTs)) > endTs) {
                // we're done with scanner
                break;
            }
            List<DimensionValue> dimensionValues = codec.getDimensionValues(rowResult.getRow());
            // At this point, we know that the record is in right time range and its dimensions matches given.
            // We try find first non-null valued dimension in the record that was not in given dimensions: we use it to form
            // next drill down suggestion
            int filledIndex = -1;
            for (int index : dimToFillIndexes) {
                // todo: it may be not efficient, if dimensionValues is not array-backed list: i.e. if access by index is
                // not fast
                DimensionValue dimensionValue = dimensionValues.get(index);
                if (dimensionValue.getValue() != null) {
                    result.add(dimensionValue);
                    filledIndex = index;
                    break;
                }
            }
            // todo: fast-forwarding (jumping) should be done on server-side (CDAP-1421)
            if (filledIndex >= 0) {
                scanner.close();
                scanner = null;
                scans++;
                if (scans > MAX_SCANS_DURING_SEARCH) {
                    break;
                }
                startRow = codec.getNextRowKey(rowResult.getRow(), filledIndex);
                scanner = timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter);
            }
        }
    } finally {
        if (scanner != null) {
            scanner.close();
        }
    }
    LOG.trace("search for dimensions completed, scans performed: {}, scanned records: {}", scans, scannedRecords);
    return result;
}

Example 15 with DimensionValue

use of io.cdap.cdap.api.dataset.lib.cube.DimensionValue in project cdap by caskdata.

the class FactTableTest method testSearch.

@Test
public void testSearch() throws Exception {
    InMemoryTableService.create("SearchEntityTable");
    InMemoryTableService.create("SearchDataTable");
    int resolution = Integer.MAX_VALUE;
    int rollTimebaseInterval = 2;
    FactTable table = new FactTable(new InMemoryMetricsTable("SearchDataTable"), new EntityTable(new InMemoryMetricsTable("SearchEntityTable")), resolution, rollTimebaseInterval);
    // aligned to start of resolution bucket
    // "/1000" because time is expected to be in seconds
    long ts = ((System.currentTimeMillis() / 1000) / resolution) * resolution;
    List<String> aggregationList = ImmutableList.of("dim1", "dim2", "dim3", "dim4");
    for (int i = 0; i < 2; i++) {
        writeInc(table, "metric-a" + i, ts + i, i, "dim1", "value1", "dim2", "value2", "dim3", "value3", "dim4", "value4");
        writeInc(table, "metric-b" + i, ts + i, i, "dim1", "value2", "dim2", "value2", "dim3", "x3", "dim4", "x4");
        writeInc(table, "metric-c" + i, ts + i, i, "dim1", "value2", "dim2", "value2", "dim3", null, "dim4", "y4");
        writeInc(table, "metric-d" + i, ts + i, i, "dim1", "value1", "dim2", "value3", "dim3", "y3", "dim4", null);
    }
    Map<String, String> slice = Maps.newHashMap();
    slice.put("dim1", "value2");
    slice.put("dim2", "value2");
    slice.put("dim3", null);
    // verify search dims
    testTagSearch(table, aggregationList, ImmutableMap.of("dim2", "value2"), ImmutableSet.of(new DimensionValue("dim1", "value1"), new DimensionValue("dim1", "value2")));
    testTagSearch(table, aggregationList, ImmutableMap.of("dim1", "value1"), ImmutableSet.of(new DimensionValue("dim2", "value2"), new DimensionValue("dim2", "value3")));
    testTagSearch(table, aggregationList, ImmutableMap.<String, String>of(), ImmutableSet.of(new DimensionValue("dim1", "value1"), new DimensionValue("dim1", "value2")));
    testTagSearch(table, aggregationList, ImmutableMap.of("dim1", "value2", "dim2", "value2"), ImmutableSet.of(new DimensionValue("dim3", "x3"), new DimensionValue("dim4", "y4")));
    testTagSearch(table, aggregationList, slice, ImmutableSet.of(new DimensionValue("dim4", "x4"), new DimensionValue("dim4", "y4")));
    testTagSearch(table, aggregationList, ImmutableMap.of("dim1", "value2", "dim2", "value3", "dim3", "y3"), ImmutableSet.<DimensionValue>of());
    // verify search metrics
    testMetricNamesSearch(table, aggregationList, ImmutableMap.of("dim1", "value1", "dim2", "value2", "dim3", "value3"), ImmutableSet.<String>of("metric-a0", "metric-a1"));
    testMetricNamesSearch(table, aggregationList, ImmutableMap.of("dim2", "value2"), ImmutableSet.<String>of("metric-a0", "metric-a1", "metric-b0", "metric-b1", "metric-c0", "metric-c1"));
    testMetricNamesSearch(table, aggregationList, slice, ImmutableSet.of("metric-b0", "metric-b1", "metric-c0", "metric-c1"));
}