Examples with TimeBin com.vmware.xenon.common.ServiceStats.TimeSeriesStats.TimeBin used on opensource projects

Example 1 with TimeBin

use of com.vmware.xenon.common.ServiceStats.TimeSeriesStats.TimeBin in project photon-model by vmware.

the class InMemoryResourceMetricService method handlePut.

@Override
public void handlePut(Operation put) {
    if (!put.hasBody()) {
        put.fail(new IllegalArgumentException("body is required"));
    }
    try {
        InMemoryResourceMetric currentState = getState(put);
        InMemoryResourceMetric updatedState = getBody(put);
        // merge the state
        for (Entry<String, TimeSeriesStats> tsStats : updatedState.timeSeriesStats.entrySet()) {
            TimeSeriesStats currentStats = currentState.timeSeriesStats.get(tsStats.getKey());
            if (currentStats == null) {
                currentState.timeSeriesStats.put(tsStats.getKey(), tsStats.getValue());
            } else {
                for (Entry<Long, TimeBin> bin : tsStats.getValue().bins.entrySet()) {
                    for (int i = 0; i < bin.getValue().count; i++) {
                        currentStats.add(TimeUnit.MILLISECONDS.toMicros(bin.getKey()), bin.getValue().avg, bin.getValue().avg);
                    }
                }
            }
        }
        setState(put, currentState);
        put.setBody(currentState).complete();
    } catch (Throwable t) {
        put.fail(t);
    }
}

Example 2 with TimeBin

use of com.vmware.xenon.common.ServiceStats.TimeSeriesStats.TimeBin in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method aggregateInMemoryMetrics.

private void aggregateInMemoryMetrics(SingleResourceStatsAggregationTaskState currentState, Map<String, SortedMap<Long, List<TimeBin>>> inMemoryStats) {
    Map<String, Map<Long, TimeBin>> aggregatedTimeBinMap = currentState.aggregatedTimeBinMap;
    for (Entry<String, SortedMap<Long, List<TimeBin>>> inMemoryStatEntry : inMemoryStats.entrySet()) {
        String metricName = inMemoryStatEntry.getKey();
        SortedMap<Long, List<TimeBin>> timeSeriesStats = inMemoryStatEntry.getValue();
        if (aggregatedTimeBinMap == null) {
            aggregatedTimeBinMap = new HashMap<>();
            currentState.aggregatedTimeBinMap = aggregatedTimeBinMap;
        }
        Map<Long, TimeBin> timeBinMap = aggregatedTimeBinMap.get(metricName);
        if (timeBinMap == null) {
            timeBinMap = new HashMap<>();
            aggregatedTimeBinMap.put(metricName, timeBinMap);
        }
        Set<AggregationType> aggregationTypes;
        for (Entry<Long, List<TimeBin>> bins : timeSeriesStats.entrySet()) {
            Long binId = bins.getKey();
            TimeBin bin = timeBinMap.get(binId);
            if (bin == null) {
                bin = new TimeBin();
            }
            // Figure out the aggregation for the given metric
            aggregationTypes = currentState.aggregations.get(stripRollupKey(metricName));
            if (aggregationTypes == null) {
                aggregationTypes = EnumSet.allOf(AggregationType.class);
            }
            for (TimeBin timeBin : bins.getValue()) {
                updateBin(bin, timeBin, aggregationTypes);
            }
            timeBinMap.put(binId, bin);
        }
    }
}

Example 3 with TimeBin

use of com.vmware.xenon.common.ServiceStats.TimeSeriesStats.TimeBin in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method processInMemoryTimeBins.

/**
 * Process in-memory time bins. This method buckets bins from time series stats appropriately
 * in the inMemoryStats data structure.
 */
private void processInMemoryTimeBins(SingleResourceStatsAggregationTaskState currentState, Map<String, SortedMap<Long, List<TimeBin>>> inMemoryStats, Entry<String, Long> metricEntry, TimeSeriesStats timeSeriesStats) {
    String metricKeyWithRollUp = metricEntry.getKey();
    String rawMetricKey = stripRollupKey(metricEntry.getKey());
    Long lastRollupTime = metricEntry.getValue();
    for (Entry<Long, TimeBin> binEntry : timeSeriesStats.bins.entrySet()) {
        // TODO VSYM-3190 - Change normalized interval boundary to beginning of the rollup period
        // Currently, xenon's time interval boundary 1 hour before than
        // photon model's aggregate metrics.
        Long binId = TimeUnit.MILLISECONDS.toMicros(binEntry.getKey());
        binId += TimeUnit.MILLISECONDS.toMicros(timeSeriesStats.binDurationMillis);
        if (binId < lastRollupTime) {
            continue;
        }
        SortedMap<Long, List<TimeBin>> bins = inMemoryStats.get(metricKeyWithRollUp);
        if (bins == null) {
            bins = new TreeMap<>();
        }
        List<TimeBin> binList = bins.get(binId);
        if (binList == null) {
            binList = new ArrayList<>();
            bins.put(binId, binList);
            inMemoryStats.put(metricKeyWithRollUp, bins);
        }
        TimeBin bin = binEntry.getValue();
        if (currentState.latestValueOnly.contains(rawMetricKey)) {
            // For latest value, we create a new time bin since we are only interested
            // in the latest data point.
            TimeBin latest = new TimeBin();
            latest.avg = latest.min = latest.max = latest.sum = bin.latest;
            latest.count = 1;
            bins.get(binId).add(latest);
        } else {
            bins.get(binId).add(bin);
        }
    }
}

Example 4 with TimeBin

use of com.vmware.xenon.common.ServiceStats.TimeSeriesStats.TimeBin in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method publishMetrics.

/**
 * Publish aggregate metric values
 */
private void publishMetrics(SingleResourceStatsAggregationTaskState currentState) {
    long expirationTime = Utils.getNowMicrosUtc() + TimeUnit.DAYS.toMicros(EXPIRATION_INTERVAL);
    List<Operation> operations = new ArrayList<>();
    Set<String> publishedKeys = new HashSet<>();
    if (!currentState.hasResources) {
        // reset the metrics to default, when no stats endpoint is available for a resource
        operations = setDefaultMetricValue(currentState, operations, publishedKeys);
    } else {
        if (currentState.aggregatedTimeBinMap == null) {
            addLastRollupTimeForMissingKeys(currentState, publishedKeys, operations);
        } else {
            for (Entry<String, Map<Long, TimeBin>> aggregateEntries : currentState.aggregatedTimeBinMap.entrySet()) {
                Map<Long, TimeBin> aggrValue = aggregateEntries.getValue();
                List<Long> keys = new ArrayList<>();
                keys.addAll(aggrValue.keySet());
                // create list of operations sorted by the timebin
                Collections.sort(keys);
                Long latestTimeKey = null;
                for (Long timeKey : keys) {
                    ResourceMetrics resourceMetrics = new ResourceMetrics();
                    resourceMetrics.entries = new HashMap<>();
                    resourceMetrics.entries.put(aggregateEntries.getKey(), aggrValue.get(timeKey).avg);
                    resourceMetrics.timestampMicrosUtc = timeKey;
                    resourceMetrics.documentSelfLink = StatsUtil.getMetricKey(currentState.resourceLink, Utils.getNowMicrosUtc());
                    resourceMetrics.documentExpirationTimeMicros = expirationTime;
                    operations.add(Operation.createPost(UriUtils.buildUri(ClusterUtil.getClusterUri(getHost(), ServiceTypeCluster.METRIC_SERVICE), ResourceMetricsService.FACTORY_LINK)).setBody(resourceMetrics));
                    publishedKeys.add(aggregateEntries.getKey());
                    latestTimeKey = timeKey;
                }
                // update the last update time as a stat
                ServiceStats.ServiceStat lastUpdateStat = new ServiceStats.ServiceStat();
                lastUpdateStat.name = aggregateEntries.getKey();
                lastUpdateStat.latestValue = latestTimeKey;
                URI inMemoryStatsUri = UriUtils.buildStatsUri(UriUtils.extendUri(ClusterUtil.getClusterUri(getHost(), ServiceTypeCluster.INVENTORY_SERVICE), currentState.resourceLink));
                operations.add(Operation.createPost(inMemoryStatsUri).setBody(lastUpdateStat));
            }
            addLastRollupTimeForMissingKeys(currentState, publishedKeys, operations);
        }
    }
    if (operations.isEmpty()) {
        // nothing to persist, just finish the task
        sendSelfPatch(currentState, TaskStage.FINISHED, null);
        return;
    }
    batchPublishMetrics(currentState, operations, 0);
}

Example 5 with TimeBin

use of com.vmware.xenon.common.ServiceStats.TimeSeriesStats.TimeBin in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method aggregateRawMetrics.

private void aggregateRawMetrics(SingleResourceStatsAggregationTaskState currentState, Map<String, List<ResourceMetrics>> rawMetricsForKey) {
    Map<String, Map<Long, TimeBin>> aggregatedTimeBinMap = currentState.aggregatedTimeBinMap;
    // comparator used to sort resource metric PODOs based on document timestamp
    Comparator<ResourceMetrics> comparator = (o1, o2) -> {
        if (o1.timestampMicrosUtc < o2.timestampMicrosUtc) {
            return -1;
        } else if (o1.timestampMicrosUtc > o2.timestampMicrosUtc) {
            return 1;
        }
        return 0;
    };
    for (Entry<String, List<ResourceMetrics>> rawMetricListEntry : rawMetricsForKey.entrySet()) {
        List<ResourceMetrics> rawMetricList = rawMetricListEntry.getValue();
        if (rawMetricList.isEmpty()) {
            continue;
        }
        String metricKeyWithRpllupSuffix = rawMetricListEntry.getKey();
        rawMetricList.sort(comparator);
        if (aggregatedTimeBinMap == null) {
            aggregatedTimeBinMap = new HashMap<>();
            currentState.aggregatedTimeBinMap = aggregatedTimeBinMap;
        }
        Map<Long, TimeBin> timeBinMap = aggregatedTimeBinMap.get(metricKeyWithRpllupSuffix);
        if (timeBinMap == null) {
            timeBinMap = new HashMap<>();
            aggregatedTimeBinMap.put(metricKeyWithRpllupSuffix, timeBinMap);
        }
        String rawMetricKey = stripRollupKey(metricKeyWithRpllupSuffix);
        Collection<ResourceMetrics> metrics = rawMetricList;
        if (currentState.latestValueOnly.contains(rawMetricKey)) {
            metrics = getLatestMetrics(rawMetricList, metricKeyWithRpllupSuffix);
        }
        Set<AggregationType> aggregationTypes;
        // iterate over the raw metric values and place it in the right time bin
        for (ResourceMetrics metric : metrics) {
            Double value = metric.entries.get(rawMetricKey);
            if (value == null) {
                continue;
            }
            // TODO VSYM-3190 - Change normalized interval boundary to beginning of the rollup period
            long binId = StatsUtil.computeIntervalEndMicros(metric.timestampMicrosUtc, lookupBinSize(metricKeyWithRpllupSuffix));
            TimeBin bin = timeBinMap.get(binId);
            if (bin == null) {
                bin = new TimeBin();
            }
            // Figure out the aggregation for the given metric
            aggregationTypes = currentState.aggregations.get(rawMetricKey);
            if (aggregationTypes == null) {
                aggregationTypes = EnumSet.allOf(AggregationType.class);
            }
            updateBin(bin, value, aggregationTypes);
            timeBinMap.put(binId, bin);
        }
    }
}