Examples with ResourceMetrics com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics used on opensource projects

Example 6 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class AzureComputeHostStatsGatherer method aggregateComputeStatsResponses.

/**
 * Aggregates stats from all the compute VMs to make up compute Host stats.
 */
private ComputeStats aggregateComputeStatsResponses(AzureStatsDataHolder statsData, List<QueryTask> items) {
    int numberOfComputeResponse = items.size();
    ComputeStats computeStats = new ComputeStats();
    computeStats.computeLink = statsData.computeHost.documentSelfLink;
    Map<String, ServiceStat> statMap = new HashMap<>();
    // Gather all the stats in a single response.
    for (QueryTask queryResult : items) {
        if (queryResult.results.documents != null) {
            for (String key : queryResult.results.documents.keySet()) {
                ResourceMetrics metric = Utils.fromJson(queryResult.results.documents.get(key), ResourceMetrics.class);
                for (Map.Entry<String, Double> entry : metric.entries.entrySet()) {
                    String metricName = entry.getKey();
                    if (statMap.containsKey(metricName)) {
                        statMap.get(metricName).latestValue += entry.getValue();
                    } else {
                        ServiceStat stat = new ServiceStat();
                        stat.latestValue = entry.getValue();
                        statMap.put(metricName, stat);
                    }
                }
            }
        }
    }
    computeStats.statValues = new ConcurrentSkipListMap<>();
    // Divide each metric value by the number of computes to get an average value.
    for (String key : statMap.keySet()) {
        ServiceStat serviceStatValue = statMap.get(key);
        serviceStatValue.unit = PhotonModelConstants.getUnitForMetric(key);
        serviceStatValue.sourceTimeMicrosUtc = Utils.getNowMicrosUtc();
        serviceStatValue.latestValue = serviceStatValue.latestValue / numberOfComputeResponse;
        computeStats.statValues.put(key, Collections.singletonList(serviceStatValue));
    }
    return computeStats;
}

Example 7 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class AzureCostStatsService method getServiceMetadata.

private void getServiceMetadata(Context context, Stages next) {
    Query.Builder builder = Query.Builder.create();
    builder.addKindFieldClause(ResourceMetrics.class);
    builder.addCompositeFieldClause(ResourceMetrics.FIELD_NAME_CUSTOM_PROPERTIES, ResourceMetrics.PROPERTY_RESOURCE_LINK, context.computeHostDesc.documentSelfLink);
    builder.addCompositeFieldClause(ResourceMetrics.FIELD_NAME_CUSTOM_PROPERTIES, PhotonModelConstants.CONTAINS_BILL_PROCESSED_TIME_STAT, Boolean.TRUE.toString());
    QueryTask.Builder queryTaskBuilder = QueryTask.Builder.createDirectTask().addOption(QueryOption.SORT).addOption(QueryOption.TOP_RESULTS).addOption(QueryOption.INCLUDE_ALL_VERSIONS).addOption(QueryOption.EXPAND_CONTENT).orderDescending(ServiceDocument.FIELD_NAME_SELF_LINK, ServiceDocumentDescription.TypeName.STRING).setResultLimit(1).setQuery(builder.build());
    if (!AzureCostConstants.SHOULD_REFRESH_INDEX) {
        queryTaskBuilder.addOption(QueryOption.DO_NOT_REFRESH);
    }
    QueryTask queryTask = queryTaskBuilder.build();
    queryTask.tenantLinks = context.computeHostDesc.tenantLinks;
    Operation queryTaskOp = QueryUtils.createQueryTaskOperation(this, queryTask, ServiceTypeCluster.METRIC_SERVICE).setExpiration(Utils.fromNowMicrosUtc(TimeUnit.SECONDS.toMicros(INTERNAL_REQUEST_TIMEOUT_SECONDS)));
    this.sendWithDeferredResult(queryTaskOp, QueryTask.class).whenComplete((response, exception) -> {
        if (exception != null) {
            handleError(context, Stages.GET_HISTORICAL_COSTS, exception, true);
            return;
        }
        if (response == null || response.results == null || response.results.documents == null) {
            context.billProcessedTimeMillis = 0;
            return;
        }
        Collection<Object> values = response.results.documents.values();
        if (!values.isEmpty()) {
            if (values.iterator().next() == null) {
                context.billProcessedTimeMillis = 0;
                return;
            }
            ResourceMetrics rawResourceMetrics = Utils.fromJson(values.iterator().next(), ResourceMetrics.class);
            context.billProcessedTimeMillis = rawResourceMetrics.entries.getOrDefault(PhotonModelConstants.CLOUD_ACCOUNT_COST_SYNC_MARKER_MILLIS, 0d).longValue();
            context.oldestBillProcessedMillis = rawResourceMetrics.entries.getOrDefault(AzureCostConstants.OLDEST_BILL_PROCESSED_MILLIS, 0d).longValue();
        }
        context.stage = next;
        handleRequest(context);
    });
}

Example 8 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class AzureCostStatsService method getStoredEaUsageCost.

private void getStoredEaUsageCost(Context context, Stages next) {
    Query.Builder builder = Query.Builder.create();
    builder.addKindFieldClause(ResourceMetrics.class);
    builder.addCompositeFieldClause(ResourceMetrics.FIELD_NAME_CUSTOM_PROPERTIES, ResourceMetrics.PROPERTY_RESOURCE_LINK, context.computeHostDesc.documentSelfLink);
    builder.addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, AzureCostConstants.USAGE_COST), QueryTask.NumericRange.createDoubleRange(0d, Double.MAX_VALUE, true, true));
    Builder queryTaskBuilder = Builder.createDirectTask().addOption(QueryOption.SORT).addOption(QueryOption.TOP_RESULTS).addOption(QueryOption.INCLUDE_ALL_VERSIONS).addOption(QueryOption.EXPAND_CONTENT).orderDescending(ResourceMetrics.FIELD_NAME_TIMESTAMP, TypeName.LONG).setResultLimit(1).setQuery(builder.build());
    if (!AzureCostConstants.SHOULD_REFRESH_INDEX) {
        queryTaskBuilder.addOption(QueryOption.DO_NOT_REFRESH);
    }
    QueryTask queryTask = queryTaskBuilder.build();
    QueryUtils.startQueryTask(this, queryTask, ServiceTypeCluster.METRIC_SERVICE).whenComplete((response, exception) -> {
        if (exception != null) {
            handleError(context, Stages.DOWNLOAD_DETAILED_BILL, exception, true);
            return;
        }
        if (response == null || response.results == null || response.results.documents == null) {
            context.storedCurrentMonthEaUsageCost = null;
            return;
        }
        Collection<Object> values = response.results.documents.values();
        if (!values.isEmpty()) {
            if (values.iterator().next() == null) {
                context.storedCurrentMonthEaUsageCost = null;
                return;
            }
            ResourceMetrics rawResourceMetrics = Utils.fromJson(values.iterator().next(), ResourceMetrics.class);
            context.storedCurrentMonthEaUsageCost = rawResourceMetrics.entries.get(AzureCostConstants.USAGE_COST);
        }
        context.stage = next;
        handleRequest(context);
    });
}

Example 9 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method getLastRollupTimeFromQuery.

private void getLastRollupTimeFromQuery(SingleResourceStatsAggregationTaskState currentState, Map<String, Long> lastUpdateMap) {
    List<Operation> operations = new ArrayList<>();
    for (String metricName : currentState.metricNames) {
        List<String> rollupKeys = buildRollupKeys(metricName);
        for (String rollupKey : rollupKeys) {
            // if the last update time was computed based on the memory stat, move on
            if (lastUpdateMap.get(rollupKey) != null) {
                continue;
            }
            String resourceId = UriUtils.getLastPathSegment(currentState.resourceLink);
            String metricSelfLink = UriUtils.buildUriPath(ResourceMetricsService.FACTORY_LINK, resourceId);
            Query.Builder builder = Query.Builder.create();
            builder.addKindFieldClause(ResourceMetrics.class);
            builder.addFieldClause(ResourceMetrics.FIELD_NAME_SELF_LINK, metricSelfLink, MatchType.PREFIX);
            builder.addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, rollupKey), NumericRange.createDoubleRange(0.0, Double.MAX_VALUE, true, true));
            QueryTask queryTask = Builder.createDirectTask().addOption(QueryOption.SORT).addOption(QueryOption.TOP_RESULTS).addOption(QueryOption.INCLUDE_ALL_VERSIONS).addOption(QueryOption.EXPAND_CONTENT).orderDescending(ServiceDocument.FIELD_NAME_SELF_LINK, TypeName.STRING).setResultLimit(1).setQuery(builder.build()).build();
            Operation op = QueryUtils.createQueryTaskOperation(this, queryTask, ServiceTypeCluster.METRIC_SERVICE);
            logInfo(() -> String.format("Invoking a query to obtain last rollup time for %s ", currentState.resourceLink));
            operations.add(op);
        }
    }
    // flood the system with lot of queries.
    if (operations.size() == 0) {
        SingleResourceStatsAggregationTaskState patchBody = new SingleResourceStatsAggregationTaskState();
        patchBody.taskInfo = TaskUtils.createTaskState(TaskStage.STARTED);
        // setting hasResources for purpose of test cases
        patchBody.hasResources = currentState.hasResources;
        patchBody.taskStage = StatsAggregationStage.INIT_RESOURCE_QUERY;
        patchBody.lastRollupTimeForMetric = lastUpdateMap;
        sendSelfPatch(patchBody);
        return;
    }
    OperationSequence opSequence = null;
    for (Operation operation : operations) {
        if (opSequence == null) {
            opSequence = OperationSequence.create(operation);
            continue;
        }
        opSequence = opSequence.next(operation);
    }
    opSequence.setCompletion((ops, failures) -> {
        if (failures != null) {
            sendSelfFailurePatch(currentState, failures.values().iterator().next().getMessage());
            return;
        }
        for (Operation operation : ops.values()) {
            QueryTask response = operation.getBody(QueryTask.class);
            for (Object obj : response.results.documents.values()) {
                ResourceMetrics resourceMetrics = Utils.fromJson(obj, ResourceMetrics.class);
                for (String metricName : resourceMetrics.entries.keySet()) {
                    lastUpdateMap.replace(metricName, resourceMetrics.timestampMicrosUtc);
                }
            }
        }
        SingleResourceStatsAggregationTaskState patchBody = new SingleResourceStatsAggregationTaskState();
        patchBody.taskInfo = TaskUtils.createTaskState(TaskStage.STARTED);
        patchBody.hasResources = currentState.hasResources;
        patchBody.taskStage = StatsAggregationStage.INIT_RESOURCE_QUERY;
        patchBody.lastRollupTimeForMetric = lastUpdateMap;
        sendSelfPatch(patchBody);
    });
    opSequence.sendWith(this);
}

Example 10 with ResourceMetrics

use of com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method getRawMetrics.

private void getRawMetrics(SingleResourceStatsAggregationTaskState currentState, QueryTask resourceQueryTask, Map<String, Set<String>> metricsToBeQueried, Map<String, SortedMap<Long, List<TimeBin>>> inMemoryStats) {
    if (metricsToBeQueried == null || metricsToBeQueried.isEmpty()) {
        aggregateMetrics(currentState, resourceQueryTask, null, inMemoryStats);
        return;
    }
    Query.Builder overallQueryBuilder = Query.Builder.create();
    for (Entry<String, Set<String>> entry : metricsToBeQueried.entrySet()) {
        String resourceId = entry.getKey();
        for (String metricKey : entry.getValue()) {
            logFine(() -> String.format("Querying raw metrics from disk for %s", metricKey));
            Long range = null;
            int binSize = 0;
            for (Entry<String, Long> metricEntry : currentState.lastRollupTimeForMetric.entrySet()) {
                if (metricEntry.getKey().startsWith(metricKey)) {
                    if (range == null || range > metricEntry.getValue()) {
                        binSize = lookupBinSize(metricEntry.getKey());
                        range = metricEntry.getValue();
                    }
                }
            }
            Query.Builder builder = Query.Builder.create(Occurance.SHOULD_OCCUR);
            builder.addKindFieldClause(ResourceMetrics.class);
            builder.addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, UriUtils.buildUriPath(ResourceMetricsService.FACTORY_LINK, resourceId), MatchType.PREFIX);
            builder.addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, metricKey), NumericRange.createDoubleRange(0.0, Double.MAX_VALUE, true, true));
            if (range != null && range != 0) {
                builder.addRangeClause(ResourceMetrics.FIELD_NAME_TIMESTAMP, NumericRange.createGreaterThanOrEqualRange(StatsUtil.computeIntervalBeginMicros(range - 1, binSize)));
            }
            overallQueryBuilder.addClause(builder.build());
        }
    }
    // create a set of rollup metric keys we are interested in and the timestamp
    // to rollup from for each
    Set<RollupMetricHolder> rollupMetricHolder = new HashSet<>();
    for (Entry<String, Long> metricEntry : currentState.lastRollupTimeForMetric.entrySet()) {
        RollupMetricHolder metric = new RollupMetricHolder();
        metric.rollupKey = metricEntry.getKey();
        if (metricEntry.getValue() != null && metricEntry.getValue() != 0) {
            metric.beginTimestampMicros = StatsUtil.computeIntervalBeginMicros(metricEntry.getValue() - 1, lookupBinSize(metricEntry.getKey()));
        }
        rollupMetricHolder.add(metric);
    }
    QueryTask task = QueryTask.Builder.createDirectTask().addOption(QueryOption.EXPAND_CONTENT).addOption(QueryOption.TOP_RESULTS).setResultLimit(RAW_METRICS_LIMIT).setQuery(overallQueryBuilder.build()).build();
    task.documentExpirationTimeMicros = Utils.getNowMicrosUtc() + QueryUtils.MINUTE_IN_MICROS;
    QueryUtils.startQueryTask(this, task, ServiceTypeCluster.METRIC_SERVICE).whenComplete((response, queryEx) -> {
        if (queryEx != null) {
            sendSelfFailurePatch(currentState, queryEx.getMessage());
            return;
        }
        Map<String, List<ResourceMetrics>> rawMetricsForKey = new HashMap<>();
        for (Object obj : response.results.documents.values()) {
            ResourceMetrics rawMetric = Utils.fromJson(obj, ResourceMetrics.class);
            for (RollupMetricHolder metric : rollupMetricHolder) {
                for (String rawMetricKey : rawMetric.entries.keySet()) {
                    if (!rawMetricKey.contains(stripRollupKey(metric.rollupKey))) {
                        continue;
                    }
                    // we want to consider raw metrics with the specified key and the appropriate timestamp
                    if ((metric.beginTimestampMicros == null || rawMetric.timestampMicrosUtc >= metric.beginTimestampMicros)) {
                        List<ResourceMetrics> rawMetricResultSet = rawMetricsForKey.get(metric.rollupKey);
                        if (rawMetricResultSet == null) {
                            rawMetricResultSet = new ArrayList<>();
                            rawMetricsForKey.put(metric.rollupKey, rawMetricResultSet);
                        }
                        rawMetricResultSet.add(rawMetric);
                    }
                }
            }
        }
        aggregateMetrics(currentState, resourceQueryTask, rawMetricsForKey, inMemoryStats);
    });
}