Search in sources :

Example 16 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class AWSCostStatsService method createAccountStats.

protected void createAccountStats(AWSCostStatsCreationContext statsData, LocalDate billMonth, AwsAccountDetailDto awsAccountDetailDto) {
    Consumer<ComputeState> accountStatsProcessor = (accountComputeState) -> {
        ComputeStats accountStats = new ComputeStats();
        accountStats.statValues = new ConcurrentHashMap<>();
        accountStats.computeLink = accountComputeState.documentSelfLink;
        if (isBillUpdated(statsData, awsAccountDetailDto)) {
            logWithContext(statsData, Level.INFO, () -> String.format("Persisting cost of Account: %s (%s) for month: %s", awsAccountDetailDto.id, accountComputeState.documentSelfLink, billMonth));
            ServiceStat costStat = createStat(AWSStatsNormalizer.getNormalizedUnitValue(DIMENSION_CURRENCY_VALUE), AWSStatsNormalizer.getNormalizedStatKeyValue(AWSConstants.COST), awsAccountDetailDto.billProcessedTimeMillis, awsAccountDetailDto.cost);
            accountStats.statValues.put(costStat.name, Collections.singletonList(costStat));
            ServiceStat otherCostsStat = createStat(AWSStatsNormalizer.getNormalizedUnitValue(DIMENSION_CURRENCY_VALUE), AWSConstants.OTHER_CHARGES, awsAccountDetailDto.billProcessedTimeMillis, awsAccountDetailDto.otherCharges);
            accountStats.statValues.put(otherCostsStat.name, Collections.singletonList(otherCostsStat));
            ServiceStat oneTimeChargesStat = createStat(AWSStatsNormalizer.getNormalizedUnitValue(DIMENSION_CURRENCY_VALUE), PhotonModelConstants.ACCOUNT_ONE_TIME_CHARGES, awsAccountDetailDto.billProcessedTimeMillis, awsAccountDetailDto.accountOneTimeCharges);
            accountStats.statValues.put(oneTimeChargesStat.name, Collections.singletonList(oneTimeChargesStat));
        }
        if (!accountStats.statValues.isEmpty()) {
            statsData.statsResponse.statsList.add(accountStats);
        }
    };
    processAccountStats(statsData, billMonth, awsAccountDetailDto, accountStatsProcessor);
    ResourceMetrics prevMarkerMetrics = statsData.accountsMarkersMap.get(awsAccountDetailDto.id);
    if (prevMarkerMetrics != null) {
        prevMarkerMetrics.entries.putAll(transformMapDataTypes(awsAccountDetailDto.lineCountPerInterval));
    }
}
Also used : AuthCredentialsServiceState(com.vmware.xenon.services.common.AuthCredentialsService.AuthCredentialsServiceState) Arrays(java.util.Arrays) DateTimeZone(org.joda.time.DateTimeZone) AWS_ACCOUNT_BILL_PROCESSED_TIME_MILLIS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_ACCOUNT_BILL_PROCESSED_TIME_MILLIS) AWSCsvBillParser(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSCsvBillParser) ServiceTypeCluster(com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster) QueryTask(com.vmware.xenon.services.common.QueryTask) ServiceDocument(com.vmware.xenon.common.ServiceDocument) ComputeType(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription.ComputeType) EndpointAllocationTaskService(com.vmware.photon.controller.model.tasks.EndpointAllocationTaskService) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState) ProgressListener(com.amazonaws.event.ProgressListener) STORAGE_TYPE_EBS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_EBS) Utils(com.vmware.xenon.common.Utils) STORAGE_TYPE_S3(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_S3) ACCOUNT_IS_AUTO_DISCOVERED(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.ACCOUNT_IS_AUTO_DISCOVERED) Map(java.util.Map) URI(java.net.URI) ProgressEventType(com.amazonaws.event.ProgressEventType) AWSMissingResourcesEnumerationService(com.vmware.photon.controller.model.adapters.awsadapter.enumeration.AWSMissingResourcesEnumerationService) Path(java.nio.file.Path) AWSClientManager(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManager) EndpointState(com.vmware.photon.controller.model.resources.EndpointService.EndpointState) PrintWriter(java.io.PrintWriter) ComputeStatsResponse(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse) ComputeStatsRequest(com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest) StatelessService(com.vmware.xenon.common.StatelessService) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) TaskManager(com.vmware.photon.controller.model.adapters.util.TaskManager) Set(java.util.Set) DiskState(com.vmware.photon.controller.model.resources.DiskService.DiskState) Occurance(com.vmware.xenon.services.common.QueryTask.Query.Occurance) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) AwsServiceDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsServiceDetailDto) Objects(java.util.Objects) List(java.util.List) AWS_LINKED_ACCOUNT_IDS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_LINKED_ACCOUNT_IDS) Stream(java.util.stream.Stream) UriUtils(com.vmware.xenon.common.UriUtils) Entry(java.util.Map.Entry) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) AwsServices(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSCsvBillParser.AwsServices) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) SingleResourceTaskCollectionStage(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceTaskCollectionStage) AwsAccountDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsAccountDetailDto) TransferManager(com.amazonaws.services.s3.transfer.TransferManager) QueryByPages(com.vmware.photon.controller.model.query.QueryUtils.QueryByPages) OperationContext(com.vmware.xenon.common.OperationContext) AWS_ACCOUNT_ID_KEY(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_ACCOUNT_ID_KEY) HashMap(java.util.HashMap) ComputeDescriptionService(com.vmware.photon.controller.model.resources.ComputeDescriptionService) GetObjectRequest(com.amazonaws.services.s3.model.GetObjectRequest) ResourceMetricsService(com.vmware.photon.controller.model.monitoring.ResourceMetricsService) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) ProgressEvent(com.amazonaws.event.ProgressEvent) HashSet(java.util.HashSet) AuthCredentialsService(com.vmware.xenon.services.common.AuthCredentialsService) Query(com.vmware.xenon.services.common.QueryTask.Query) UriPaths(com.vmware.photon.controller.model.UriPaths) CollectionUtils(org.apache.commons.collections.CollectionUtils) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) BiConsumer(java.util.function.BiConsumer) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) AwsClientType(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AwsClientType) StatsUtil(com.vmware.photon.controller.model.tasks.monitoring.StatsUtil) ExecutorService(java.util.concurrent.ExecutorService) AdapterUtils(com.vmware.photon.controller.model.adapters.util.AdapterUtils) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) ResourceState(com.vmware.photon.controller.model.resources.ResourceState) Files(java.nio.file.Files) StringWriter(java.io.StringWriter) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) IOException(java.io.IOException) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) AwsResourceDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsResourceDetailDto) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) LocalDate(org.joda.time.LocalDate) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) Paths(java.nio.file.Paths) ClusterUtil(com.vmware.photon.controller.model.util.ClusterUtil) AWSStatsNormalizer(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSStatsNormalizer) ComputeStateWithDescription(com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription) PhotonModelConstants(com.vmware.photon.controller.model.constants.PhotonModelConstants) AWSClientManagerFactory(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManagerFactory) ServiceDocumentDescription(com.vmware.xenon.common.ServiceDocumentDescription) Collections(java.util.Collections) OperationJoin(com.vmware.xenon.common.OperationJoin) PhotonModelUriUtils.createInventoryUri(com.vmware.photon.controller.model.util.PhotonModelUriUtils.createInventoryUri) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Example 17 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class AWSCostStatsService method createStatsForAwsService.

private Map<String, List<ServiceStat>> createStatsForAwsService(AwsServiceDetailDto serviceDetailDto) {
    String currencyUnit = AWSStatsNormalizer.getNormalizedUnitValue(DIMENSION_CURRENCY_VALUE);
    // remove any spaces in the service name.
    String serviceCode = serviceDetailDto.id.replaceAll(" ", "");
    Map<String, List<ServiceStat>> stats = new HashMap<>();
    // Create stats for hourly resource cost
    List<ServiceStat> serviceStats = new ArrayList<>();
    String serviceResourceCostMetric = String.format(AWSConstants.SERVICE_RESOURCE_COST, serviceCode);
    for (Entry<Long, Double> cost : serviceDetailDto.directCosts.entrySet()) {
        if (cost.getValue() > 0) {
            ServiceStat resourceCostStat = createStat(currencyUnit, serviceResourceCostMetric, cost.getKey(), cost.getValue());
            serviceStats.add(resourceCostStat);
        }
    }
    if (!serviceStats.isEmpty()) {
        stats.put(serviceResourceCostMetric, serviceStats);
    }
    // Create stats for hourly other costs
    serviceStats = new ArrayList<>();
    String serviceOtherCostMetric = String.format(AWSConstants.SERVICE_OTHER_COST, serviceCode);
    for (Entry<Long, Double> cost : serviceDetailDto.otherCosts.entrySet()) {
        if (cost.getValue() > 0) {
            ServiceStat otherCostStat = createStat(currencyUnit, serviceOtherCostMetric, cost.getKey(), cost.getValue());
            serviceStats.add(otherCostStat);
        }
    }
    if (!serviceStats.isEmpty()) {
        stats.put(serviceOtherCostMetric, serviceStats);
    }
    // Create stats for monthly other costs
    serviceStats = new ArrayList<>();
    String serviceMonthlyOtherCostMetric = String.format(AWSConstants.SERVICE_MONTHLY_OTHER_COST, serviceCode);
    for (Entry<Long, Double> cost : serviceDetailDto.remainingCosts.entrySet()) {
        if (cost.getValue() > 0) {
            ServiceStat monthlyOtherCostStat = createStat(currencyUnit, serviceMonthlyOtherCostMetric, cost.getKey(), cost.getValue());
            serviceStats.add(monthlyOtherCostStat);
        }
    }
    if (!serviceStats.isEmpty()) {
        stats.put(serviceMonthlyOtherCostMetric, serviceStats);
    }
    // Create stats for monthly reserved recurring instance costs
    serviceStats = new ArrayList<>();
    String serviceReservedRecurringCostMetric = String.format(AWSConstants.SERVICE_RESERVED_RECURRING_COST, serviceCode);
    for (Entry<Long, Double> cost : serviceDetailDto.reservedRecurringCosts.entrySet()) {
        if (cost.getValue() > 0) {
            ServiceStat recurringCostStat = createStat(currencyUnit, serviceReservedRecurringCostMetric, cost.getKey(), cost.getValue());
            serviceStats.add(recurringCostStat);
        }
    }
    if (!serviceStats.isEmpty()) {
        stats.put(serviceReservedRecurringCostMetric, serviceStats);
    }
    return stats;
}
Also used : ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList)

Example 18 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class AWSMockStatsService method getMockStats.

/**
 * Gets mock EC2 statistics.
 *
 * @param statsData The context object for mock stats.
 * @param metricNames The metrics names to gather stats for.
 */
private void getMockStats(AWSMockStatsDataHolder statsData, String[] metricNames) {
    for (String metricName : metricNames) {
        // start feeding mock stats data
        if (MOCK_STATS_MAP.get(metricName) != null) {
            logFine(() -> String.format("Retrieving %s mock metric from AWS", metricName));
            List<ServiceStat> statValue = MOCK_STATS_MAP.get(metricName);
            statsData.statsResponse.statValues.put(AWSStatsNormalizer.getNormalizedStatKeyValue(metricName), statValue);
        }
    }
    SingleResourceStatsCollectionTaskState respBody = new SingleResourceStatsCollectionTaskState();
    statsData.statsResponse.computeLink = statsData.computeState.documentSelfLink;
    respBody.statsAdapterReference = UriUtils.buildUri(getHost(), SELF_LINK);
    respBody.taskStage = SingleResourceTaskCollectionStage.valueOf(statsData.statsRequest.nextStage);
    respBody.statsList = new ArrayList<>();
    respBody.statsList.add(statsData.statsResponse);
    this.sendRequest(Operation.createPatch(statsData.statsRequest.taskReference).setBody(respBody));
}
Also used : ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState)

Example 19 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class LongRunEndToEndAzureStatsAggregation method checkInMemoryStatsPresent.

/**
 * Perform check to verify that every compute resource has stats generated for all the metric
 * keys using which stats aggregation was performed on resources.
 */
private void checkInMemoryStatsPresent(ServiceDocumentQueryResult res) {
    for (Map.Entry<String, Object> resourceMap : res.documents.entrySet()) {
        ServiceStats resStats = this.host.getServiceState(null, ServiceStats.class, UriUtils.buildStatsUri(createServiceURI(host, null, resourceMap.getKey())));
        int statCount = 0;
        for (ServiceStat stat : resStats.entries.values()) {
            for (String key : AzureTestUtil.getMetricNames()) {
                if (stat.name.startsWith(key)) {
                    statCount++;
                    break;
                }
            }
        }
        // after stats aggregation all resources should have all metrics.
        assertEquals("Did not find in-memory stats", AzureTestUtil.getMetricNames().size(), statCount);
    }
}
Also used : ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ServiceStats(com.vmware.xenon.common.ServiceStats) Map(java.util.Map)

Example 20 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class TestAzureEnumerationTask method issueStatsRequest.

private void issueStatsRequest(String selfLink, boolean isComputeHost) throws Throwable {
    // spin up a stateless service that acts as the parent link to patch back to
    StatelessService parentService = new StatelessService() {

        @Override
        public void handleRequest(Operation op) {
            if (op.getAction() == Action.PATCH) {
                if (!TestAzureEnumerationTask.this.isMock) {
                    ComputeStatsResponse resp = op.getBody(ComputeStatsResponse.class);
                    if (resp == null) {
                        TestAzureEnumerationTask.this.host.failIteration(new IllegalStateException("response was null."));
                        return;
                    }
                    if (resp.statsList.size() != 1) {
                        TestAzureEnumerationTask.this.host.failIteration(new IllegalStateException("response size was incorrect."));
                        return;
                    }
                    if (resp.statsList.get(0).statValues.size() == 0) {
                        TestAzureEnumerationTask.this.host.failIteration(new IllegalStateException("incorrect number of metrics received."));
                        return;
                    }
                    if (!resp.statsList.get(0).computeLink.equals(selfLink)) {
                        TestAzureEnumerationTask.this.host.failIteration(new IllegalStateException("Incorrect resourceReference returned."));
                        return;
                    }
                    // Verify all the stats are obtained
                    verifyStats(resp, isComputeHost);
                    // Persist stats on Verification Host for testing the computeHost stats.
                    URI persistStatsUri = UriUtils.buildUri(getHost(), ResourceMetricsService.FACTORY_LINK);
                    ResourceMetricsService.ResourceMetrics resourceMetric = new ResourceMetricsService.ResourceMetrics();
                    resourceMetric.documentSelfLink = StatsUtil.getMetricKey(selfLink, Utils.getNowMicrosUtc());
                    resourceMetric.entries = new HashMap<>();
                    resourceMetric.timestampMicrosUtc = Utils.getNowMicrosUtc();
                    for (String key : resp.statsList.get(0).statValues.keySet()) {
                        List<ServiceStat> stats = resp.statsList.get(0).statValues.get(key);
                        for (ServiceStat stat : stats) {
                            if (stat == null) {
                                continue;
                            }
                            resourceMetric.entries.put(key, stat.latestValue);
                        }
                    }
                    TestAzureEnumerationTask.this.host.sendRequest(Operation.createPost(persistStatsUri).setReferer(TestAzureEnumerationTask.this.host.getUri()).setBodyNoCloning(resourceMetric));
                }
                TestAzureEnumerationTask.this.host.completeIteration();
            }
        }
    };
    String servicePath = UUID.randomUUID().toString();
    Operation startOp = Operation.createPost(UriUtils.buildUri(this.host, servicePath));
    this.host.startService(startOp, parentService);
    ComputeStatsRequest statsRequest = new ComputeStatsRequest();
    statsRequest.resourceReference = UriUtils.buildUri(this.host, selfLink);
    statsRequest.nextStage = SingleResourceTaskCollectionStage.UPDATE_STATS.name();
    statsRequest.isMockRequest = this.isMock;
    statsRequest.taskReference = UriUtils.buildUri(this.host, servicePath);
    this.host.sendAndWait(Operation.createPatch(UriUtils.buildUri(this.host, AzureUriPaths.AZURE_STATS_ADAPTER)).setBody(statsRequest).setReferer(this.host.getUri()));
}
Also used : ResourceMetricsService(com.vmware.photon.controller.model.monitoring.ResourceMetricsService) StatelessService(com.vmware.xenon.common.StatelessService) Operation(com.vmware.xenon.common.Operation) AzureTestUtil.randomString(com.vmware.photon.controller.model.adapters.azure.instance.AzureTestUtil.randomString) URI(java.net.URI) ComputeStatsResponse(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ComputeStatsRequest(com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest)

Aggregations

ServiceStat (com.vmware.xenon.common.ServiceStats.ServiceStat)37 ArrayList (java.util.ArrayList)21 ComputeStats (com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats)16 SingleResourceStatsCollectionTaskState (com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState)15 List (java.util.List)11 Map (java.util.Map)11 ComputeStatsRequest (com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest)10 ResourceMetrics (com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics)10 ComputeState (com.vmware.photon.controller.model.resources.ComputeService.ComputeState)9 Operation (com.vmware.xenon.common.Operation)9 QueryTask (com.vmware.xenon.services.common.QueryTask)9 HashMap (java.util.HashMap)9 ServiceStats (com.vmware.xenon.common.ServiceStats)8 UriUtils (com.vmware.xenon.common.UriUtils)8 URI (java.net.URI)8 PhotonModelConstants (com.vmware.photon.controller.model.constants.PhotonModelConstants)7 ComputeStateWithDescription (com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription)7 ServiceDocument (com.vmware.xenon.common.ServiceDocument)7 StatelessService (com.vmware.xenon.common.StatelessService)7 Utils (com.vmware.xenon.common.Utils)7