Examples with ResourceMetrics com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics used on opensource projects

Example 26 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method getLatestMetrics.

/**
 * Returns the latest metrics from the raw metrics list. Since the list contains all raw metrics
 * across multiple resource we iterate on the list and pick the latest metric for each resource
 * per bin depending on the metric key.
 *
 * TODO VSYM-2481: Add custom mock stats adapter based test for this.
 */
private Collection<ResourceMetrics> getLatestMetrics(List<ResourceMetrics> metrics, String metricKeyWithInterval) {
    if (metrics.isEmpty()) {
        return Collections.emptyList();
    }
    // Metric link to map of latest value per bin. For example:
    // /monitoring/metrics/<resource-id>_<key1> -> 1474070400000000, <latest-value-of-key1-in-this-time-bucket>
    // /monitoring/metrics/<resource-id>_<key2> -> 1474070400000000, <latest-value-of-key2-in-this-time-bucket>
    Map<String, Map<Long, ResourceMetrics>> metricsByLatestValuePerInterval = new HashMap<>();
    for (ResourceMetrics metric : metrics) {
        String metricKey = stripRollupKey(metricKeyWithInterval);
        Map<Long, ResourceMetrics> metricsByIntervalEndTime = metricsByLatestValuePerInterval.get(metricKey);
        Double value = metric.entries.get(metricKey);
        if (value == null) {
            continue;
        }
        // TODO VSYM-3190 - Change normalized interval boundary to beginning of the rollup period
        long binId = StatsUtil.computeIntervalEndMicros(metric.timestampMicrosUtc, lookupBinSize(metricKeyWithInterval));
        if (metricsByIntervalEndTime == null) {
            metricsByIntervalEndTime = new HashMap<>();
            metricsByIntervalEndTime.put(binId, metric);
            metricsByLatestValuePerInterval.put(metricKey, metricsByIntervalEndTime);
            continue;
        }
        ResourceMetrics existingMetric = metricsByIntervalEndTime.get(binId);
        Double existingValue = null;
        if (existingMetric != null) {
            existingValue = existingMetric.entries.get(metricKey);
        }
        if (existingValue == null || existingMetric.timestampMicrosUtc < metric.timestampMicrosUtc) {
            metricsByIntervalEndTime.put(binId, metric);
        }
    }
    // Gather all latest values
    List<ResourceMetrics> result = new ArrayList<>();
    for (Map<Long, ResourceMetrics> metricsByIntervalEndTime : metricsByLatestValuePerInterval.values()) {
        result.addAll(metricsByIntervalEndTime.values());
    }
    return result;
}

Example 27 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method publishMetrics.

/**
 * Publish aggregate metric values
 */
private void publishMetrics(SingleResourceStatsAggregationTaskState currentState) {
    long expirationTime = Utils.getNowMicrosUtc() + TimeUnit.DAYS.toMicros(EXPIRATION_INTERVAL);
    List<Operation> operations = new ArrayList<>();
    Set<String> publishedKeys = new HashSet<>();
    if (!currentState.hasResources) {
        // reset the metrics to default, when no stats endpoint is available for a resource
        operations = setDefaultMetricValue(currentState, operations, publishedKeys);
    } else {
        if (currentState.aggregatedTimeBinMap == null) {
            addLastRollupTimeForMissingKeys(currentState, publishedKeys, operations);
        } else {
            for (Entry<String, Map<Long, TimeBin>> aggregateEntries : currentState.aggregatedTimeBinMap.entrySet()) {
                Map<Long, TimeBin> aggrValue = aggregateEntries.getValue();
                List<Long> keys = new ArrayList<>();
                keys.addAll(aggrValue.keySet());
                // create list of operations sorted by the timebin
                Collections.sort(keys);
                Long latestTimeKey = null;
                for (Long timeKey : keys) {
                    ResourceMetrics resourceMetrics = new ResourceMetrics();
                    resourceMetrics.entries = new HashMap<>();
                    resourceMetrics.entries.put(aggregateEntries.getKey(), aggrValue.get(timeKey).avg);
                    resourceMetrics.timestampMicrosUtc = timeKey;
                    resourceMetrics.documentSelfLink = StatsUtil.getMetricKey(currentState.resourceLink, Utils.getNowMicrosUtc());
                    resourceMetrics.documentExpirationTimeMicros = expirationTime;
                    operations.add(Operation.createPost(UriUtils.buildUri(ClusterUtil.getClusterUri(getHost(), ServiceTypeCluster.METRIC_SERVICE), ResourceMetricsService.FACTORY_LINK)).setBody(resourceMetrics));
                    publishedKeys.add(aggregateEntries.getKey());
                    latestTimeKey = timeKey;
                }
                // update the last update time as a stat
                ServiceStats.ServiceStat lastUpdateStat = new ServiceStats.ServiceStat();
                lastUpdateStat.name = aggregateEntries.getKey();
                lastUpdateStat.latestValue = latestTimeKey;
                URI inMemoryStatsUri = UriUtils.buildStatsUri(UriUtils.extendUri(ClusterUtil.getClusterUri(getHost(), ServiceTypeCluster.INVENTORY_SERVICE), currentState.resourceLink));
                operations.add(Operation.createPost(inMemoryStatsUri).setBody(lastUpdateStat));
            }
            addLastRollupTimeForMissingKeys(currentState, publishedKeys, operations);
        }
    }
    if (operations.isEmpty()) {
        // nothing to persist, just finish the task
        sendSelfPatch(currentState, TaskStage.FINISHED, null);
        return;
    }
    batchPublishMetrics(currentState, operations, 0);
}

Example 28 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method aggregateRawMetrics.

private void aggregateRawMetrics(SingleResourceStatsAggregationTaskState currentState, Map<String, List<ResourceMetrics>> rawMetricsForKey) {
    Map<String, Map<Long, TimeBin>> aggregatedTimeBinMap = currentState.aggregatedTimeBinMap;
    // comparator used to sort resource metric PODOs based on document timestamp
    Comparator<ResourceMetrics> comparator = (o1, o2) -> {
        if (o1.timestampMicrosUtc < o2.timestampMicrosUtc) {
            return -1;
        } else if (o1.timestampMicrosUtc > o2.timestampMicrosUtc) {
            return 1;
        }
        return 0;
    };
    for (Entry<String, List<ResourceMetrics>> rawMetricListEntry : rawMetricsForKey.entrySet()) {
        List<ResourceMetrics> rawMetricList = rawMetricListEntry.getValue();
        if (rawMetricList.isEmpty()) {
            continue;
        }
        String metricKeyWithRpllupSuffix = rawMetricListEntry.getKey();
        rawMetricList.sort(comparator);
        if (aggregatedTimeBinMap == null) {
            aggregatedTimeBinMap = new HashMap<>();
            currentState.aggregatedTimeBinMap = aggregatedTimeBinMap;
        }
        Map<Long, TimeBin> timeBinMap = aggregatedTimeBinMap.get(metricKeyWithRpllupSuffix);
        if (timeBinMap == null) {
            timeBinMap = new HashMap<>();
            aggregatedTimeBinMap.put(metricKeyWithRpllupSuffix, timeBinMap);
        }
        String rawMetricKey = stripRollupKey(metricKeyWithRpllupSuffix);
        Collection<ResourceMetrics> metrics = rawMetricList;
        if (currentState.latestValueOnly.contains(rawMetricKey)) {
            metrics = getLatestMetrics(rawMetricList, metricKeyWithRpllupSuffix);
        }
        Set<AggregationType> aggregationTypes;
        // iterate over the raw metric values and place it in the right time bin
        for (ResourceMetrics metric : metrics) {
            Double value = metric.entries.get(rawMetricKey);
            if (value == null) {
                continue;
            }
            // TODO VSYM-3190 - Change normalized interval boundary to beginning of the rollup period
            long binId = StatsUtil.computeIntervalEndMicros(metric.timestampMicrosUtc, lookupBinSize(metricKeyWithRpllupSuffix));
            TimeBin bin = timeBinMap.get(binId);
            if (bin == null) {
                bin = new TimeBin();
            }
            // Figure out the aggregation for the given metric
            aggregationTypes = currentState.aggregations.get(rawMetricKey);
            if (aggregationTypes == null) {
                aggregationTypes = EnumSet.allOf(AggregationType.class);
            }
            updateBin(bin, value, aggregationTypes);
            timeBinMap.put(binId, bin);
        }
    }
}

Example 29 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class SingleResourceStatsCollectionTaskService method populateLastCollectionTimeFromPersistenceStore.

/**
 * Queries the metric for the last successful run and sets that value in the compute stats request.
 * This value is used to determine the window size for which the stats collection happens from the provider.
 */
private void populateLastCollectionTimeFromPersistenceStore(SingleResourceStatsCollectionTaskState currentState, ComputeStatsRequest computeStatsRequest, URI patchUri, List<String> tenantLinks) {
    String statsAdapterLink = getAdapterLinkFromURI(patchUri);
    String lastSuccessfulRunMetricKey = getLastCollectionMetricKeyForAdapterLink(statsAdapterLink, false);
    Query.Builder builder = Query.Builder.create();
    builder.addKindFieldClause(ResourceMetrics.class);
    builder.addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, UriUtils.buildUriPath(ResourceMetricsService.FACTORY_LINK, UriUtils.getLastPathSegment(currentState.computeLink)), MatchType.PREFIX);
    builder.addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, lastSuccessfulRunMetricKey), NumericRange.createDoubleRange(Double.MIN_VALUE, Double.MAX_VALUE, true, true));
    QueryTask task = QueryTask.Builder.createDirectTask().addOption(QueryOption.SORT).orderDescending(ServiceDocument.FIELD_NAME_SELF_LINK, TypeName.STRING).addOption(QueryOption.TOP_RESULTS).addOption(QueryOption.INCLUDE_ALL_VERSIONS).setResultLimit(1).addOption(QueryOption.EXPAND_CONTENT).setQuery(builder.build()).build();
    task.tenantLinks = tenantLinks;
    QueryUtils.startQueryTask(this, task, ServiceTypeCluster.METRIC_SERVICE).whenComplete((responseTask, e) -> {
        if (e != null) {
            logSevere("Could not get the last collection time from persisted metrics: %s", Utils.toString(e));
            // Still continue calling into the adapter if the last known time for
            // successful collection is not known
            sendStatsRequestToAdapter(currentState, patchUri, computeStatsRequest);
            return;
        }
        // while sending the request to the adapter.
        if (responseTask.results.documentCount > 0) {
            Object rawMetricObj = responseTask.results.documents.get(responseTask.results.documentLinks.get(0));
            ResourceMetrics rawMetrics = Utils.fromJson(rawMetricObj, ResourceMetrics.class);
            computeStatsRequest.lastCollectionTimeMicrosUtc = rawMetrics.timestampMicrosUtc;
        }
        sendStatsRequestToAdapter(currentState, patchUri, computeStatsRequest);
    });
}

Example 30 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class SingleResourceStatsCollectionTaskService method populateResourceMetrics.

private void populateResourceMetrics(List<ResourceMetrics> metricsList, String metricName, ServiceStat serviceStat, String computeLink, long expirationTime, Map<String, String> props) {
    if (Double.isNaN(serviceStat.latestValue)) {
        return;
    }
    ResourceMetrics metricsObjToUpdate = null;
    for (ResourceMetrics metricsObj : metricsList) {
        if (metricsObj.documentSelfLink.startsWith(UriUtils.getLastPathSegment(computeLink)) && metricsObj.timestampMicrosUtc.equals(serviceStat.sourceTimeMicrosUtc)) {
            metricsObjToUpdate = metricsObj;
            break;
        }
    }
    if (metricsObjToUpdate == null) {
        metricsObjToUpdate = new ResourceMetrics();
        metricsObjToUpdate.documentSelfLink = StatsUtil.getMetricKey(computeLink, Utils.getNowMicrosUtc());
        metricsObjToUpdate.entries = new HashMap<>();
        metricsObjToUpdate.timestampMicrosUtc = serviceStat.sourceTimeMicrosUtc;
        metricsObjToUpdate.documentExpirationTimeMicros = expirationTime;
        metricsObjToUpdate.customProperties = new HashMap<>();
        metricsObjToUpdate.customProperties.put(ResourceMetrics.PROPERTY_RESOURCE_LINK, computeLink);
        metricsList.add(metricsObjToUpdate);
    }
    if (props != null) {
        metricsObjToUpdate.customProperties.putAll(props);
    }
    metricsObjToUpdate.entries.put(metricName, serviceStat.latestValue);
}