Search in sources :

Example 11 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class SingleResourceStatsCollectionTaskService method populateLastCollectionTimeForMetricsInStatsRequest.

/**
 * Gets the last collection for a compute for a given adapter URI.
 * As a first step, the in memory stats for the compute are queried and if the metric
 * for the last collection time is found, then the timestamp for that is returned.
 *
 * Else, the ResoureMetric table is queried and the latest version of the metric is used
 * to determine the last collection time for the stats.
 */
private void populateLastCollectionTimeForMetricsInStatsRequest(SingleResourceStatsCollectionTaskState currentState, ComputeStatsRequest computeStatsRequest, URI patchUri, List<String> tenantLinks) {
    URI computeStatsUri = UriUtils.buildStatsUri(UriUtils.extendUri(ClusterUtil.getClusterUri(getHost(), ServiceTypeCluster.INVENTORY_SERVICE), currentState.computeLink));
    Operation.createGet(computeStatsUri).setCompletion((o, e) -> {
        if (e != null) {
            logSevere(() -> String.format("Could not get the last collection time from" + " in memory stats: %s", Utils.toString(e)));
            // get the value from the persisted store.
            populateLastCollectionTimeFromPersistenceStore(currentState, computeStatsRequest, patchUri, tenantLinks);
            return;
        }
        ServiceStats serviceStats = o.getBody(ServiceStats.class);
        String statsAdapterLink = getAdapterLinkFromURI(patchUri);
        String lastSuccessfulRunMetricKey = getLastCollectionMetricKeyForAdapterLink(statsAdapterLink, true);
        if (serviceStats.entries.containsKey(lastSuccessfulRunMetricKey)) {
            ServiceStat lastRunStat = serviceStats.entries.get(lastSuccessfulRunMetricKey);
            computeStatsRequest.lastCollectionTimeMicrosUtc = lastRunStat.sourceTimeMicrosUtc;
            sendStatsRequestToAdapter(currentState, patchUri, computeStatsRequest);
        } else {
            populateLastCollectionTimeFromPersistenceStore(currentState, computeStatsRequest, patchUri, tenantLinks);
        }
    }).sendWith(this);
}
Also used : Service(com.vmware.xenon.common.Service) ServiceTypeCluster(com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster) QueryTask(com.vmware.xenon.services.common.QueryTask) ServiceDocument(com.vmware.xenon.common.ServiceDocument) AggregationType(com.vmware.xenon.common.ServiceStats.TimeSeriesStats.AggregationType) MatchType(com.vmware.xenon.services.common.QueryTask.QueryTerm.MatchType) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState) Utils(com.vmware.xenon.common.Utils) TaskFactoryService(com.vmware.xenon.services.common.TaskFactoryService) Map(java.util.Map) URI(java.net.URI) EnumSet(java.util.EnumSet) ComputeDescription(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription) ComputeStatsRequest(com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest) ServiceStats(com.vmware.xenon.common.ServiceStats) List(java.util.List) TimeSeriesStats(com.vmware.xenon.common.ServiceStats.TimeSeriesStats) UriUtils(com.vmware.xenon.common.UriUtils) Entry(java.util.Map.Entry) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) TaskState(com.vmware.xenon.common.TaskState) TaskOption(com.vmware.photon.controller.model.tasks.TaskOption) FactoryService(com.vmware.xenon.common.FactoryService) NumericRange(com.vmware.xenon.services.common.QueryTask.NumericRange) TaskService(com.vmware.xenon.services.common.TaskService) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) TaskUtils(com.vmware.photon.controller.model.tasks.TaskUtils) InMemoryResourceMetric(com.vmware.photon.controller.model.monitoring.InMemoryResourceMetricService.InMemoryResourceMetric) HashMap(java.util.HashMap) PhotonModelUtils(com.vmware.photon.controller.model.resources.util.PhotonModelUtils) ResourceMetricsService(com.vmware.photon.controller.model.monitoring.ResourceMetricsService) ArrayList(java.util.ArrayList) ServiceUriPaths(com.vmware.xenon.services.common.ServiceUriPaths) Query(com.vmware.xenon.services.common.QueryTask.Query) UriPaths(com.vmware.photon.controller.model.UriPaths) PropertyUsageOption(com.vmware.xenon.common.ServiceDocumentDescription.PropertyUsageOption) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) OperationSequence(com.vmware.xenon.common.OperationSequence) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) TypeName(com.vmware.xenon.common.ServiceDocumentDescription.TypeName) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) TaskStage(com.vmware.xenon.common.TaskState.TaskStage) TimeUnit(java.util.concurrent.TimeUnit) InMemoryResourceMetricService(com.vmware.photon.controller.model.monitoring.InMemoryResourceMetricService) ClusterUtil(com.vmware.photon.controller.model.util.ClusterUtil) ComputeStateWithDescription(com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription) PhotonModelConstants(com.vmware.photon.controller.model.constants.PhotonModelConstants) Comparator(java.util.Comparator) QuerySpecification(com.vmware.xenon.services.common.QueryTask.QuerySpecification) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ServiceStats(com.vmware.xenon.common.ServiceStats) URI(java.net.URI)

Example 12 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class MockStatsAdapter method handleRequest.

@Override
public void handleRequest(Operation op) {
    if (!op.hasBody()) {
        op.fail(new IllegalArgumentException("body is required"));
        return;
    }
    switch(op.getAction()) {
        case PATCH:
            op.complete();
            ComputeStatsRequest statsRequest = op.getBody(ComputeStatsRequest.class);
            SingleResourceStatsCollectionTaskState statsResponse = new SingleResourceStatsCollectionTaskState();
            Map<String, List<ServiceStat>> statValues = new HashMap<>();
            double currentCounter = this.counter.incrementAndGet();
            ServiceStat key1 = new ServiceStat();
            key1.latestValue = currentCounter;
            long timestampMicros = Utils.getNowMicrosUtc();
            key1.sourceTimeMicrosUtc = timestampMicros;
            key1.unit = UNIT_1;
            statValues.put(KEY_1, Collections.singletonList(key1));
            sendBatchResponse(statsRequest, statsResponse, statValues, false);
            statValues.clear();
            ServiceStat key2 = new ServiceStat();
            key2.latestValue = currentCounter;
            key2.sourceTimeMicrosUtc = timestampMicros;
            key2.unit = UNIT_2;
            statValues.put(KEY_2, Collections.singletonList(key2));
            sendBatchResponse(statsRequest, statsResponse, statValues, true);
            break;
        default:
            super.handleRequest(op);
    }
}
Also used : ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ComputeStatsRequest(com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest) HashMap(java.util.HashMap) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState) ArrayList(java.util.ArrayList) List(java.util.List)

Example 13 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class StatsCollectionTaskServiceTest method testCustomStatsAdapterPrecedence.

@Test
public void testCustomStatsAdapterPrecedence() throws Throwable {
    ResourcePoolState rpState = new ResourcePoolState();
    rpState.name = UUID.randomUUID().toString();
    ResourcePoolState rpReturnState = postServiceSynchronously(ResourcePoolService.FACTORY_LINK, rpState, ResourcePoolState.class);
    ComputeDescription desc = new ComputeDescription();
    desc.name = rpState.name;
    desc.statsAdapterReference = UriUtils.buildUri(this.host, MockStatsAdapter.SELF_LINK);
    desc.statsAdapterReferences = new HashSet<>(Arrays.asList(UriUtils.buildUri(this.host, "/foo"), UriUtils.buildUri(this.host, CustomStatsAdapter.SELF_LINK)));
    ComputeDescription descReturnState = postServiceSynchronously(ComputeDescriptionService.FACTORY_LINK, desc, ComputeDescription.class);
    ComputeState computeState = new ComputeState();
    computeState.name = rpState.name;
    computeState.descriptionLink = descReturnState.documentSelfLink;
    computeState.resourcePoolLink = rpReturnState.documentSelfLink;
    List<String> computeLinks = new ArrayList<>();
    for (int i = 0; i < this.numResources; i++) {
        ComputeState res = postServiceSynchronously(ComputeService.FACTORY_LINK, computeState, ComputeState.class);
        computeLinks.add(res.documentSelfLink);
    }
    // create a stats collection scheduler task
    StatsCollectionTaskState statCollectionState = new StatsCollectionTaskState();
    statCollectionState.resourcePoolLink = rpReturnState.documentSelfLink;
    statCollectionState.statsAdapterReference = UriUtils.buildUri(this.host, CustomStatsAdapter.SELF_LINK);
    statCollectionState.options = EnumSet.of(TaskOption.SELF_DELETE_ON_COMPLETION);
    ScheduledTaskState statsCollectionTaskState = new ScheduledTaskState();
    statsCollectionTaskState.factoryLink = StatsCollectionTaskService.FACTORY_LINK;
    statsCollectionTaskState.initialStateJson = Utils.toJson(statCollectionState);
    statsCollectionTaskState.intervalMicros = TimeUnit.MILLISECONDS.toMicros(500);
    statsCollectionTaskState = postServiceSynchronously(ScheduledTaskService.FACTORY_LINK, statsCollectionTaskState, ScheduledTaskState.class);
    ServiceDocumentQueryResult res = this.host.getFactoryState(UriUtils.buildExpandLinksQueryUri(UriUtils.buildUri(this.host, ScheduledTaskService.FACTORY_LINK)));
    assertTrue(res.documents.size() > 0);
    // get stats from resources
    for (int i = 0; i < computeLinks.size(); i++) {
        String statsUriPath = UriUtils.buildUriPath(computeLinks.get(i), ServiceHost.SERVICE_URI_SUFFIX_STATS);
        this.host.waitFor("Error waiting for stats", () -> {
            ServiceStats resStats = getServiceSynchronously(statsUriPath, ServiceStats.class);
            boolean returnStatus = false;
            // populated correctly.
            for (ServiceStat stat : resStats.entries.values()) {
                // host.log(Level.INFO, "*****%s", stat.name);
                if (stat.name.startsWith(UriUtils.getLastPathSegment(CustomStatsAdapter.SELF_LINK))) {
                    returnStatus = true;
                    break;
                }
            }
            return returnStatus;
        });
    }
    // clean up
    deleteServiceSynchronously(statsCollectionTaskState.documentSelfLink);
}
Also used : ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) ResourcePoolState(com.vmware.photon.controller.model.resources.ResourcePoolService.ResourcePoolState) ComputeDescription(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription) ArrayList(java.util.ArrayList) ServiceDocumentQueryResult(com.vmware.xenon.common.ServiceDocumentQueryResult) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ServiceStats(com.vmware.xenon.common.ServiceStats) ScheduledTaskState(com.vmware.photon.controller.model.tasks.ScheduledTaskService.ScheduledTaskState) StatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.StatsCollectionTaskService.StatsCollectionTaskState) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState) BaseModelTest(com.vmware.photon.controller.model.helpers.BaseModelTest) Test(org.junit.Test)

Example 14 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class AWSCostStatsService method createStat.

private ServiceStat createStat(String unit, String name, Long timestamp, Number value) {
    ServiceStat stat = new ServiceStat();
    stat.latestValue = value.doubleValue();
    stat.sourceTimeMicrosUtc = TimeUnit.MILLISECONDS.toMicros(timestamp);
    stat.unit = unit;
    stat.name = name;
    return stat;
}
Also used : ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat)

Example 15 with ServiceStat

use of com.vmware.xenon.common.ServiceStats.ServiceStat in project photon-model by vmware.

the class AWSCostStatsService method createServiceStatsForAccount.

protected void createServiceStatsForAccount(AWSCostStatsCreationContext statsData, LocalDate billMonth, AwsAccountDetailDto awsAccountDetailDto) {
    Consumer<ComputeState> serviceStatsProcessor = (accountComputeState) -> {
        ComputeStats awsServiceStats = new ComputeStats();
        awsServiceStats.statValues = new ConcurrentHashMap<>();
        awsServiceStats.computeLink = accountComputeState.documentSelfLink;
        awsServiceStats.addCustomProperty(PhotonModelConstants.DOES_CONTAIN_SERVICE_STATS, Boolean.TRUE.toString());
        for (AwsServiceDetailDto serviceDetailDto : awsAccountDetailDto.serviceDetailsMap.values()) {
            Map<String, List<ServiceStat>> statsForAwsService = createStatsForAwsService(serviceDetailDto);
            awsServiceStats.statValues.putAll(statsForAwsService);
        }
        if (!awsServiceStats.statValues.isEmpty()) {
            statsData.statsResponse.statsList.add(awsServiceStats);
        }
    };
    insertEC2ServiceDetail(awsAccountDetailDto);
    processAccountStats(statsData, billMonth, awsAccountDetailDto, serviceStatsProcessor);
}
Also used : AuthCredentialsServiceState(com.vmware.xenon.services.common.AuthCredentialsService.AuthCredentialsServiceState) Arrays(java.util.Arrays) DateTimeZone(org.joda.time.DateTimeZone) AWS_ACCOUNT_BILL_PROCESSED_TIME_MILLIS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_ACCOUNT_BILL_PROCESSED_TIME_MILLIS) AWSCsvBillParser(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSCsvBillParser) ServiceTypeCluster(com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster) QueryTask(com.vmware.xenon.services.common.QueryTask) ServiceDocument(com.vmware.xenon.common.ServiceDocument) ComputeType(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription.ComputeType) EndpointAllocationTaskService(com.vmware.photon.controller.model.tasks.EndpointAllocationTaskService) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState) ProgressListener(com.amazonaws.event.ProgressListener) STORAGE_TYPE_EBS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_EBS) Utils(com.vmware.xenon.common.Utils) STORAGE_TYPE_S3(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_S3) ACCOUNT_IS_AUTO_DISCOVERED(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.ACCOUNT_IS_AUTO_DISCOVERED) Map(java.util.Map) URI(java.net.URI) ProgressEventType(com.amazonaws.event.ProgressEventType) AWSMissingResourcesEnumerationService(com.vmware.photon.controller.model.adapters.awsadapter.enumeration.AWSMissingResourcesEnumerationService) Path(java.nio.file.Path) AWSClientManager(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManager) EndpointState(com.vmware.photon.controller.model.resources.EndpointService.EndpointState) PrintWriter(java.io.PrintWriter) ComputeStatsResponse(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse) ComputeStatsRequest(com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest) StatelessService(com.vmware.xenon.common.StatelessService) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) TaskManager(com.vmware.photon.controller.model.adapters.util.TaskManager) Set(java.util.Set) DiskState(com.vmware.photon.controller.model.resources.DiskService.DiskState) Occurance(com.vmware.xenon.services.common.QueryTask.Query.Occurance) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) AwsServiceDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsServiceDetailDto) Objects(java.util.Objects) List(java.util.List) AWS_LINKED_ACCOUNT_IDS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_LINKED_ACCOUNT_IDS) Stream(java.util.stream.Stream) UriUtils(com.vmware.xenon.common.UriUtils) Entry(java.util.Map.Entry) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) AwsServices(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSCsvBillParser.AwsServices) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) SingleResourceTaskCollectionStage(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceTaskCollectionStage) AwsAccountDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsAccountDetailDto) TransferManager(com.amazonaws.services.s3.transfer.TransferManager) QueryByPages(com.vmware.photon.controller.model.query.QueryUtils.QueryByPages) OperationContext(com.vmware.xenon.common.OperationContext) AWS_ACCOUNT_ID_KEY(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_ACCOUNT_ID_KEY) HashMap(java.util.HashMap) ComputeDescriptionService(com.vmware.photon.controller.model.resources.ComputeDescriptionService) GetObjectRequest(com.amazonaws.services.s3.model.GetObjectRequest) ResourceMetricsService(com.vmware.photon.controller.model.monitoring.ResourceMetricsService) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) ProgressEvent(com.amazonaws.event.ProgressEvent) HashSet(java.util.HashSet) AuthCredentialsService(com.vmware.xenon.services.common.AuthCredentialsService) Query(com.vmware.xenon.services.common.QueryTask.Query) UriPaths(com.vmware.photon.controller.model.UriPaths) CollectionUtils(org.apache.commons.collections.CollectionUtils) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) BiConsumer(java.util.function.BiConsumer) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) AwsClientType(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AwsClientType) StatsUtil(com.vmware.photon.controller.model.tasks.monitoring.StatsUtil) ExecutorService(java.util.concurrent.ExecutorService) AdapterUtils(com.vmware.photon.controller.model.adapters.util.AdapterUtils) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) ResourceState(com.vmware.photon.controller.model.resources.ResourceState) Files(java.nio.file.Files) StringWriter(java.io.StringWriter) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) IOException(java.io.IOException) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) AwsResourceDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsResourceDetailDto) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) LocalDate(org.joda.time.LocalDate) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) Paths(java.nio.file.Paths) ClusterUtil(com.vmware.photon.controller.model.util.ClusterUtil) AWSStatsNormalizer(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSStatsNormalizer) ComputeStateWithDescription(com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription) PhotonModelConstants(com.vmware.photon.controller.model.constants.PhotonModelConstants) AWSClientManagerFactory(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManagerFactory) ServiceDocumentDescription(com.vmware.xenon.common.ServiceDocumentDescription) Collections(java.util.Collections) OperationJoin(com.vmware.xenon.common.OperationJoin) PhotonModelUriUtils.createInventoryUri(com.vmware.photon.controller.model.util.PhotonModelUriUtils.createInventoryUri) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) AwsServiceDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsServiceDetailDto) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap)

Aggregations

ServiceStat (com.vmware.xenon.common.ServiceStats.ServiceStat)37 ArrayList (java.util.ArrayList)21 ComputeStats (com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats)16 SingleResourceStatsCollectionTaskState (com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState)15 List (java.util.List)11 Map (java.util.Map)11 ComputeStatsRequest (com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest)10 ResourceMetrics (com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics)10 ComputeState (com.vmware.photon.controller.model.resources.ComputeService.ComputeState)9 Operation (com.vmware.xenon.common.Operation)9 QueryTask (com.vmware.xenon.services.common.QueryTask)9 HashMap (java.util.HashMap)9 ServiceStats (com.vmware.xenon.common.ServiceStats)8 UriUtils (com.vmware.xenon.common.UriUtils)8 URI (java.net.URI)8 PhotonModelConstants (com.vmware.photon.controller.model.constants.PhotonModelConstants)7 ComputeStateWithDescription (com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription)7 ServiceDocument (com.vmware.xenon.common.ServiceDocument)7 StatelessService (com.vmware.xenon.common.StatelessService)7 Utils (com.vmware.xenon.common.Utils)7