Examples with FactTable io.cdap.cdap.data2.dataset2.lib.timeseries.FactTable used on opensource projects

Example 11 with FactTable

use of io.cdap.cdap.data2.dataset2.lib.timeseries.FactTable in project cdap by caskdata.

the class DefaultMetricDatasetFactory method getOrCreateFactTable.

// todo: figure out roll time based on resolution from config? See DefaultMetricsTableFactory for example
@Override
public FactTable getOrCreateFactTable(int resolution) {
    String v3TableName = cConf.get(Constants.Metrics.METRICS_TABLE_PREFIX, Constants.Metrics.DEFAULT_METRIC_V3_TABLE_PREFIX) + ".ts." + resolution;
    int ttl = cConf.getInt(Constants.Metrics.RETENTION_SECONDS + "." + resolution + ".seconds", -1);
    TableProperties.Builder props = TableProperties.builder();
    // don't add TTL for MAX_RESOLUTION table. CDAP-1626
    if (ttl > 0 && resolution != Integer.MAX_VALUE) {
        props.setTTL(ttl);
    }
    // for efficient counters
    props.setReadlessIncrementSupport(true);
    // configuring pre-splits
    props.add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(FactTable.getSplits(DefaultMetricStore.AGGREGATIONS.size())));
    // Disable auto split
    props.add(HBaseTableAdmin.SPLIT_POLICY, cConf.get(Constants.Metrics.METRICS_TABLE_HBASE_SPLIT_POLICY));
    MetricsTable table = getOrCreateResolutionMetricsTable(v3TableName, props, resolution);
    return new FactTable(table, entityTable.get(), resolution, getRollTime(resolution));
}

Example 12 with FactTable

use of io.cdap.cdap.data2.dataset2.lib.timeseries.FactTable in project cdap by caskdata.

the class DefaultCube method add.

@Override
public void add(Collection<? extends CubeFact> facts) {
    List<Fact> toWrite = Lists.newArrayList();
    int dimValuesCount = 0;
    for (CubeFact fact : facts) {
        for (Map.Entry<String, ? extends Aggregation> aggEntry : aggregations.entrySet()) {
            Aggregation agg = aggEntry.getValue();
            AggregationAlias aggregationAlias = null;
            if (aggregationAliasMap.containsKey(aggEntry.getKey())) {
                aggregationAlias = aggregationAliasMap.get(aggEntry.getKey());
            }
            if (agg.accept(fact)) {
                List<DimensionValue> dimensionValues = Lists.newArrayList();
                for (String dimensionName : agg.getDimensionNames()) {
                    String dimensionValueKey = aggregationAlias == null ? dimensionName : aggregationAlias.getAlias(dimensionName);
                    dimensionValues.add(new DimensionValue(dimensionName, fact.getDimensionValues().get(dimensionValueKey)));
                    dimValuesCount++;
                }
                toWrite.add(new Fact(fact.getTimestamp(), dimensionValues, fact.getMeasurements()));
            }
        }
    }
    for (FactTable table : resolutionToFactTable.values()) {
        table.add(toWrite);
    }
    incrementMetric("cube.cubeFact.add.request.count", 1);
    incrementMetric("cube.cubeFact.added.count", facts.size());
    incrementMetric("cube.tsFact.created.count", toWrite.size());
    incrementMetric("cube.tsFact.created.dimValues.count", dimValuesCount);
    incrementMetric("cube.tsFact.added.count", toWrite.size() * resolutionToFactTable.size());
}

Example 13 with FactTable

use of io.cdap.cdap.data2.dataset2.lib.timeseries.FactTable in project cdap by caskdata.

the class DefaultCube method findMeasureNames.

@Override
public Collection<String> findMeasureNames(CubeExploreQuery query) {
    LOG.trace("Searching for measures, query: {}", query);
    // In each aggregation that matches given dimensions, try to find measure names
    SortedSet<String> result = Sets.newTreeSet();
    // todo: the passed query should have map instead
    LinkedHashMap<String, String> slice = Maps.newLinkedHashMap();
    for (DimensionValue dimensionValue : query.getDimensionValues()) {
        slice.put(dimensionValue.getName(), dimensionValue.getValue());
    }
    FactTable table = resolutionToFactTable.get(query.getResolution());
    for (Aggregation agg : aggregations.values()) {
        if (agg.getDimensionNames().containsAll(slice.keySet())) {
            result.addAll(table.findMeasureNames(agg.getDimensionNames(), slice, query.getStartTs(), query.getEndTs()));
        }
    }
    return result;
}

Example 14 with FactTable

use of io.cdap.cdap.data2.dataset2.lib.timeseries.FactTable in project cdap by caskdata.

the class DefaultCube method delete.

@Override
public void delete(CubeDeleteQuery query) {
    // this may be very inefficient and its better to use TTL, this is to only support existing old functionality.
    List<DimensionValue> dimensionValues = Lists.newArrayList();
    // use the dimension values of the aggregation to delete entries in all the fact-tables.
    for (Aggregation agg : aggregations.values()) {
        if (agg.getDimensionNames().containsAll(query.getDimensionValues().keySet())) {
            dimensionValues.clear();
            for (String dimensionName : agg.getDimensionNames()) {
                dimensionValues.add(new DimensionValue(dimensionName, query.getDimensionValues().get(dimensionName)));
            }
            FactTable factTable = resolutionToFactTable.get(query.getResolution());
            FactScan scan = new FactScan(query.getStartTs(), query.getEndTs(), query.getMeasureNames(), dimensionValues);
            factTable.delete(scan);
        }
    }
}

Example 15 with FactTable

use of io.cdap.cdap.data2.dataset2.lib.timeseries.FactTable in project cdap by caskdata.

the class DefaultCube method findDimensionValues.

@Override
public Collection<DimensionValue> findDimensionValues(CubeExploreQuery query) {
    LOG.trace("Searching for next-level context, query: {}", query);
    // In each aggregation that matches given dimensions, try to fill in value in a single null-valued given dimension.
    // NOTE: that we try to fill in first value that is non-null-valued in a stored record
    // (see FactTable#findSingleDimensionValue)
    SortedSet<DimensionValue> result = Sets.newTreeSet(DIMENSION_VALUE_COMPARATOR);
    // todo: the passed query should have map instead
    LinkedHashMap<String, String> slice = Maps.newLinkedHashMap();
    for (DimensionValue dimensionValue : query.getDimensionValues()) {
        slice.put(dimensionValue.getName(), dimensionValue.getValue());
    }
    FactTable table = resolutionToFactTable.get(query.getResolution());
    for (Aggregation agg : aggregations.values()) {
        if (agg.getDimensionNames().containsAll(slice.keySet())) {
            result.addAll(table.findSingleDimensionValue(agg.getDimensionNames(), slice, query.getStartTs(), query.getEndTs()));
        }
    }
    return result;
}