Examples with FactScan co.cask.cdap.data2.dataset2.lib.timeseries.FactScan used on opensource projects

Example 6 with FactScan

use of co.cask.cdap.data2.dataset2.lib.timeseries.FactScan in project cdap by caskdata.

the class DefaultCube method query.

@Override
public Collection<TimeSeries> query(CubeQuery query) {
    /*
      CubeQuery example: "dataset read ops for app per dataset". Or:

      SELECT count('read.ops')                                           << measure name and type
      FROM aggregation1.1min_resolution                                  << aggregation and resolution
      GROUP BY dataset,                                                  << groupByDimensions
      WHERE namespace='ns1' AND app='myApp' AND program='myFlow' AND     << dimensionValues
            ts>=1423370200 AND ts{@literal<}1423398198                   << startTs and endTs
      LIMIT 100                                                          << limit

      Execution:

      1) (optional, if aggregation to query in is not provided) find aggregation to supply results

      Here, we need aggregation that has following dimensions: 'namespace', 'app', 'program', 'dataset'.

      Ideally (to reduce the scan range), 'dataset' should be in the end, other dimensions as close to the beginning
      as possible, and minimal number of other "unspecified" dimensions.

      Let's say we found aggregation: 'namespace', 'app', 'program', 'instance', 'dataset'

      2) build a scan in the aggregation

      For scan we set "any" into the dimension values that aggregation has but query doesn't define value for:

      'namespace'='ns1', 'app'='myApp', 'program'='myFlow', 'instance'=*, 'dataset'=*

      Plus specified measure & aggregation?:

      'measureName'='read.ops'
      'measureType'='COUNTER'

      3) While scanning build a table: dimension values -> time -> value. Use measureType as values aggregate
         function if needed.
    */
    incrementMetric("cube.query.request.count", 1);
    if (!resolutionToFactTable.containsKey(query.getResolution())) {
        incrementMetric("cube.query.request.failure.count", 1);
        throw new IllegalArgumentException("There's no data aggregated for specified resolution to satisfy the query: " + query.toString());
    }
    // 1) find aggregation to query
    Aggregation agg;
    String aggName;
    if (query.getAggregation() != null) {
        aggName = query.getAggregation();
        agg = aggregations.get(query.getAggregation());
        if (agg == null) {
            incrementMetric("cube.query.request.failure.count", 1);
            throw new IllegalArgumentException(String.format("Specified aggregation %s is not found in cube aggregations: %s", query.getAggregation(), aggregations.keySet().toString()));
        }
    } else {
        ImmutablePair<String, Aggregation> aggregation = findAggregation(query);
        if (aggregation == null) {
            incrementMetric("cube.query.request.failure.count", 1);
            throw new IllegalArgumentException("There's no data aggregated for specified dimensions " + "to satisfy the query: " + query.toString());
        }
        agg = aggregation.getSecond();
        aggName = aggregation.getFirst();
    }
    // tell how many queries end up querying specific pre-aggregated views and resolutions
    incrementMetric("cube.query.agg." + aggName + ".count", 1);
    incrementMetric("cube.query.res." + query.getResolution() + ".count", 1);
    // 2) build a scan for a query
    List<DimensionValue> dimensionValues = Lists.newArrayList();
    for (String dimensionName : agg.getDimensionNames()) {
        // if not defined in query, will be set as null, which means "any"
        dimensionValues.add(new DimensionValue(dimensionName, query.getDimensionValues().get(dimensionName)));
    }
    FactScan scan = new FactScan(query.getStartTs(), query.getEndTs(), query.getMeasurements().keySet(), dimensionValues);
    // 3) execute scan query
    FactTable table = resolutionToFactTable.get(query.getResolution());
    FactScanner scanner = table.scan(scan);
    Table<Map<String, String>, String, Map<Long, Long>> resultMap = getTimeSeries(query, scanner);
    incrementMetric("cube.query.request.success.count", 1);
    incrementMetric("cube.query.result.size", resultMap.size());
    Collection<TimeSeries> timeSeries = convertToQueryResult(query, resultMap);
    incrementMetric("cube.query.result.timeseries.count", timeSeries.size());
    return timeSeries;
}

Example 7 with FactScan

use of co.cask.cdap.data2.dataset2.lib.timeseries.FactScan in project cdap by caskdata.

the class FactTableTest method testQuery.

@Test
public void testQuery() throws Exception {
    InMemoryTableService.create("QueryEntityTable");
    InMemoryTableService.create("QueryDataTable");
    int resolution = 10;
    int rollTimebaseInterval = 2;
    FactTable table = new FactTable(new InMemoryMetricsTable("QueryDataTable"), new EntityTable(new InMemoryMetricsTable("QueryEntityTable")), resolution, rollTimebaseInterval);
    // aligned to start of resolution bucket
    // "/1000" because time is expected to be in seconds
    long ts = ((System.currentTimeMillis() / 1000) / resolution) * resolution;
    for (int i = 0; i < 3; i++) {
        for (int k = 1; k < 3; k++) {
            // note: "+i" to ts here and below doesn't affect results, just to confirm
            //       that data points are rounded to the resolution
            writeInc(table, "metric" + k, ts + i * resolution + i, k + i, "dim1", "value1", "dim2", "value2");
            writeInc(table, "metric" + k, ts + i * resolution + i, 2 * k + i, "dim1", "value2", "dim2", "value2");
            writeInc(table, "metric" + k, ts + i * resolution + i, 3 * k + i, "dim1", "value2", "dim2", "value1");
            writeInc(table, "metric" + k, ts + i * resolution + i, 4 * k + i, "dim1", "value1", "dim2", "value3");
            // null value in dim matches only fuzzy ("any")
            writeInc(table, "metric" + k, ts + i * resolution + i, 5 * k + i, "dim1", null, "dim2", "value3");
        }
    }
    Table<String, List<DimensionValue>, List<TimeValue>> expected;
    FactScan scan;
    for (int i = 1; i < 3; i++) {
        // all time points
        scan = new FactScan(ts - resolution, ts + 3 * resolution, "metric" + i, dimValues("dim1", "value1", "dim2", "value2"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1, i + 2));
        assertScan(table, expected, scan);
        // time points since second interval
        scan = new FactScan(ts + resolution, ts + 3 * resolution, "metric" + i, dimValues("dim1", "value1", "dim2", "value2"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        assertScan(table, expected, scan);
        // time points before third interval
        scan = new FactScan(ts - resolution, ts + resolution, "metric" + i, dimValues("dim1", "value1", "dim2", "value2"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1));
        assertScan(table, expected, scan);
        // time points for fuzzy dim2 since second interval
        scan = new FactScan(ts + resolution, ts + 3 * resolution, // null stands for any
        "metric" + i, dimValues("dim1", "value1", "dim2", null));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts + resolution, resolution, 4 * i + 1, 4 * i + 2));
        assertScan(table, expected, scan);
        // time points for fuzzy dim1 before third interval
        scan = new FactScan(ts - resolution, ts + resolution, // null stands for any
        "metric" + i, dimValues("dim1", null, "dim2", "value3"));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts, resolution, 4 * i, 4 * i + 1));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts, resolution, 5 * i, 5 * i + 1));
        assertScan(table, expected, scan);
        // time points for both fuzzy dims before third interval
        scan = new FactScan(ts - resolution, ts + resolution, // null stands for any
        "metric" + i, dimValues("dim1", null, "dim2", null));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value1"), timeValues(ts, resolution, 3 * i, 3 * i + 1));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value2"), timeValues(ts, resolution, 2 * i, 2 * i + 1));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts, resolution, 4 * i, 4 * i + 1));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts, resolution, 5 * i, 5 * i + 1));
        assertScan(table, expected, scan);
        // time points for both fuzzy dims since third interval
        scan = new FactScan(ts + resolution, ts + 3 * resolution, // null stands for any
        "metric" + i, dimValues("dim1", null, "dim2", null));
        expected = HashBasedTable.create();
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value1"), timeValues(ts + resolution, resolution, 3 * i + 1, 3 * i + 2));
        expected.put("metric" + i, dimValues("dim1", "value2", "dim2", "value2"), timeValues(ts + resolution, resolution, 2 * i + 1, 2 * i + 2));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts + resolution, resolution, 4 * i + 1, 4 * i + 2));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts + resolution, resolution, 5 * i + 1, 5 * i + 2));
        assertScan(table, expected, scan);
    }
    // all time points
    scan = new FactScan(ts - resolution, ts + 3 * resolution, dimValues("dim1", "value1", "dim2", "value2"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1, i + 2));
    }
    assertScan(table, expected, scan);
    // time points since second interval
    scan = new FactScan(ts + resolution, ts + 3 * resolution, dimValues("dim1", "value1", "dim2", "value2"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
    }
    assertScan(table, expected, scan);
    // time points before third interval
    scan = new FactScan(ts - resolution, ts + resolution, dimValues("dim1", "value1", "dim2", "value2"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts, resolution, i, i + 1));
    }
    assertScan(table, expected, scan);
    // time points for fuzzy dim2 since second interval
    scan = new FactScan(ts + resolution, ts + 3 * resolution, dimValues("dim1", "value1", "dim2", null));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value2"), timeValues(ts + resolution, resolution, i + 1, i + 2));
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts + resolution, resolution, 4 * i + 1, 4 * i + 2));
    }
    assertScan(table, expected, scan);
    // time points for fuzzy dim1 before third interval (very important case - caught some bugs)
    scan = new FactScan(ts - resolution, ts + resolution, dimValues("dim1", null, "dim2", "value3"));
    expected = HashBasedTable.create();
    for (int i = 1; i < 3; i++) {
        expected.put("metric" + i, dimValues("dim1", "value1", "dim2", "value3"), timeValues(ts, resolution, 4 * i, 4 * i + 1));
        expected.put("metric" + i, dimValues("dim1", null, "dim2", "value3"), timeValues(ts, resolution, 5 * i, 5 * i + 1));
    }
    assertScan(table, expected, scan);
}

Example 8 with FactScan

use of co.cask.cdap.data2.dataset2.lib.timeseries.FactScan in project cdap by caskdata.

the class DefaultCube method delete.

@Override
public void delete(CubeDeleteQuery query) {
    //this may be very inefficient and its better to use TTL, this is to only support existing old functionality.
    List<DimensionValue> dimensionValues = Lists.newArrayList();
    // use the dimension values of the aggregation to delete entries in all the fact-tables.
    for (Aggregation agg : aggregations.values()) {
        if (agg.getDimensionNames().containsAll(query.getDimensionValues().keySet())) {
            dimensionValues.clear();
            for (String dimensionName : agg.getDimensionNames()) {
                dimensionValues.add(new DimensionValue(dimensionName, query.getDimensionValues().get(dimensionName)));
            }
            FactTable factTable = resolutionToFactTable.get(query.getResolution());
            FactScan scan = new FactScan(query.getStartTs(), query.getEndTs(), query.getMeasureNames(), dimensionValues);
            factTable.delete(scan);
        }
    }
}