Search in sources :

Example 11 with ServiceStats

use of com.vmware.xenon.common.ServiceStats in project photon-model by vmware.

the class StatsCollectionTaskServiceTest method testStatsCollection.

private void testStatsCollection(boolean testOnCluster) throws Throwable {
    VerificationHost metricHost = null;
    if (testOnCluster) {
        metricHost = this.setupMetricHost();
    }
    // Use this.host if metricHost is null.
    VerificationHost verificationHost = (metricHost == null ? this.host : metricHost);
    // create a metric host
    // create a compute description for all the computes
    ComputeDescription cDesc = new ComputeDescription();
    cDesc.name = UUID.randomUUID().toString();
    cDesc.statsAdapterReference = UriUtils.buildUri(this.host, MockStatsAdapter.SELF_LINK);
    ComputeDescription descReturnState = postServiceSynchronously(ComputeDescriptionService.FACTORY_LINK, cDesc, ComputeDescription.class);
    // create multiple computes
    ComputeState computeState = new ComputeState();
    computeState.name = UUID.randomUUID().toString();
    computeState.descriptionLink = descReturnState.documentSelfLink;
    List<String> computeLinks = new ArrayList<>(this.numResources);
    for (int i = 0; i < this.numResources; i++) {
        ComputeState res = postServiceSynchronously(ComputeService.FACTORY_LINK, computeState, ComputeState.class);
        computeLinks.add(res.documentSelfLink);
    }
    // create a resource pool including all the created computes
    ResourcePoolState rpState = new ResourcePoolState();
    rpState.name = UUID.randomUUID().toString();
    rpState.properties = EnumSet.of(ResourcePoolProperty.ELASTIC);
    rpState.query = Query.Builder.create().addKindFieldClause(ComputeState.class).addInClause(ServiceDocument.FIELD_NAME_SELF_LINK, computeLinks).build();
    ResourcePoolState rpReturnState = postServiceSynchronously(ResourcePoolService.FACTORY_LINK, rpState, ResourcePoolState.class);
    // create a stats collection scheduler task
    StatsCollectionTaskState statCollectionState = new StatsCollectionTaskState();
    statCollectionState.resourcePoolLink = rpReturnState.documentSelfLink;
    statCollectionState.options = EnumSet.of(TaskOption.SELF_DELETE_ON_COMPLETION);
    ScheduledTaskState statsCollectionTaskState = new ScheduledTaskState();
    statsCollectionTaskState.factoryLink = StatsCollectionTaskService.FACTORY_LINK;
    statsCollectionTaskState.initialStateJson = Utils.toJson(statCollectionState);
    statsCollectionTaskState.intervalMicros = TimeUnit.SECONDS.toMicros(2);
    statsCollectionTaskState = postServiceSynchronously(ScheduledTaskService.FACTORY_LINK, statsCollectionTaskState, ScheduledTaskState.class);
    ServiceDocumentQueryResult res = this.host.getFactoryState(UriUtils.buildExpandLinksQueryUri(UriUtils.buildUri(this.host, ScheduledTaskService.FACTORY_LINK)));
    assertTrue(res.documents.size() > 0);
    // the last successful collection time should be populated as an in memory stat.
    for (int i = 0; i < this.numResources; i++) {
        String statsUriPath = UriUtils.buildUriPath(computeLinks.get(i), ServiceHost.SERVICE_URI_SUFFIX_STATS);
        this.host.waitFor("Error waiting for in memory stats", () -> {
            ServiceStats resStats = getServiceSynchronously(statsUriPath, ServiceStats.class);
            boolean returnStatus = false;
            for (ServiceStat stat : resStats.entries.values()) {
                if (stat.latestValue > 0) {
                    returnStatus = true;
                    break;
                }
            }
            return returnStatus;
        });
    }
    host.log(Level.INFO, "Successfully verified that all the last collection time is available in memory.");
    // persisted at a per metric level along with the last collection run time
    for (String computeLink : computeLinks) {
        ResourceMetrics metric = getResourceMetrics(verificationHost, computeLink, MockStatsAdapter.KEY_1);
        assertNotNull("The resource metric for" + MockStatsAdapter.KEY_1 + " should not be null ", metric);
        assertEquals(metric.entries.size(), 1);
        assertEquals(metric.customProperties.get("prop1"), "val1");
        ResourceMetrics metric2 = getResourceMetrics(verificationHost, computeLink, MockStatsAdapter.KEY_2);
        assertNotNull("The resource metric for" + MockStatsAdapter.KEY_2 + "should not be null ", metric2);
        assertEquals(metric2.entries.size(), 1);
        String lastSuccessfulRunMetricKey = UriUtils.getLastPathSegment(MockStatsAdapter.SELF_LINK) + StatsUtil.SEPARATOR + PhotonModelConstants.LAST_SUCCESSFUL_STATS_COLLECTION_TIME;
        ResourceMetrics metricLastRun = getResourceMetrics(verificationHost, computeLink, lastSuccessfulRunMetricKey);
        assertNotNull("The resource metric for" + lastSuccessfulRunMetricKey + " should not be null ", metricLastRun);
    }
    host.log(Level.INFO, "Successfully verified that the required resource metrics are persisted in the resource metrics table");
    // Verify sorted order of the metrics versions by timestamp
    for (String computeLink : computeLinks) {
        // get all versions
        QueryTask qt = QueryTask.Builder.createDirectTask().addOption(QueryOption.EXPAND_CONTENT).addOption(QueryOption.SORT).orderAscending(ServiceDocument.FIELD_NAME_SELF_LINK, TypeName.STRING).setQuery(Query.Builder.create().addKindFieldClause(ResourceMetrics.class).addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, UriUtils.buildUriPath(ResourceMetricsService.FACTORY_LINK, UriUtils.getLastPathSegment(computeLink)), MatchType.PREFIX).addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, MockStatsAdapter.KEY_1), NumericRange.createDoubleRange(Double.MIN_VALUE, Double.MAX_VALUE, true, true)).build()).build();
        verificationHost.createQueryTaskService(qt, false, true, qt, null);
        ResourceMetrics prevMetric = null;
        for (String documentLink : qt.results.documentLinks) {
            ResourceMetrics metric = Utils.fromJson(qt.results.documents.get(documentLink), ResourceMetrics.class);
            if (prevMetric == null) {
                prevMetric = metric;
                continue;
            }
            assertTrue(prevMetric.timestampMicrosUtc < metric.timestampMicrosUtc);
        }
    }
    // verify that the aggregation tasks have been deleted
    this.host.waitFor("Timeout waiting for task to expire", () -> {
        ServiceDocumentQueryResult collectRes = this.host.getFactoryState(UriUtils.buildUri(this.host, StatsCollectionTaskService.FACTORY_LINK));
        if (collectRes.documentLinks.size() == 0) {
            return true;
        }
        return false;
    });
    if (testOnCluster) {
        this.cleanUpMetricHost(metricHost);
    }
}
Also used : ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) ResourcePoolState(com.vmware.photon.controller.model.resources.ResourcePoolService.ResourcePoolState) ComputeDescription(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription) ArrayList(java.util.ArrayList) VerificationHost(com.vmware.xenon.common.test.VerificationHost) ServiceDocumentQueryResult(com.vmware.xenon.common.ServiceDocumentQueryResult) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) ServiceStats(com.vmware.xenon.common.ServiceStats) QueryTask(com.vmware.xenon.services.common.QueryTask) ScheduledTaskState(com.vmware.photon.controller.model.tasks.ScheduledTaskService.ScheduledTaskState) StatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.StatsCollectionTaskService.StatsCollectionTaskState) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState)

Example 12 with ServiceStats

use of com.vmware.xenon.common.ServiceStats in project photon-model by vmware.

the class GCPTestUtil method enumerateResources.

/**
 * Method to run enumeration on GCP endpoint.
 * @param host The test service host.
 * @param peerURI The peer uri.
 * @param enumTask The enumeration task to run.
 * @param testCase The test case name.
 * @throws Throwable Exception when running enumeration tasks.
 */
public static void enumerateResources(VerificationHost host, URI peerURI, ResourceEnumerationTaskState enumTask, String testCase) throws Throwable {
    // Perform resource enumeration on the GCP end point.
    // Pass the references to the GCP compute host.
    host.log("Performing resource enumeration");
    ResourceEnumerationTaskState enumTaskState = performResourceEnumeration(host, peerURI, enumTask);
    // Wait for the enumeration task to be completed.
    host.waitForFinishedTask(ResourceEnumerationTaskState.class, createServiceURI(host, peerURI, enumTaskState.documentSelfLink));
    host.log("\n==%s==Total Time Spent in Enumeration==\n", testCase + getVMCount(host, peerURI));
    ServiceStats enumerationStats = host.getServiceState(null, ServiceStats.class, UriUtils.buildStatsUri(createServiceURI(host, peerURI, GCPEnumerationAdapterService.SELF_LINK)));
    host.log(Utils.toJsonHtml(enumerationStats));
}
Also used : ResourceEnumerationTaskState(com.vmware.photon.controller.model.tasks.ResourceEnumerationTaskService.ResourceEnumerationTaskState) ServiceStats(com.vmware.xenon.common.ServiceStats)

Example 13 with ServiceStats

use of com.vmware.xenon.common.ServiceStats in project photon-model by vmware.

the class SingleResourceStatsAggregationTaskService method getLastRollupTime.

private void getLastRollupTime(SingleResourceStatsAggregationTaskState currentState) {
    Map<String, Long> lastUpdateMap = new HashMap<>();
    for (String metricName : currentState.metricNames) {
        List<String> rollupKeys = buildRollupKeys(metricName);
        for (String rollupKey : rollupKeys) {
            lastUpdateMap.put(rollupKey, null);
        }
    }
    // Lookup last rollup time from in memory stats - /<resource-link>/stats
    URI statsUri = UriUtils.buildStatsUri(UriUtils.extendUri(ClusterUtil.getClusterUri(getHost(), ServiceTypeCluster.INVENTORY_SERVICE), currentState.resourceLink));
    sendRequest(Operation.createGet(statsUri).setCompletion((o, e) -> {
        if (e != null) {
            logWarning(() -> String.format("Could not get stats for resource: %s," + " error: %s", currentState.resourceLink, e.getMessage()));
            // get the value based on a query.
            getLastRollupTimeFromQuery(currentState, lastUpdateMap);
            return;
        }
        ServiceStats serviceStats = o.getBody(ServiceStats.class);
        lastUpdateMap.keySet().stream().filter(rollupKey -> serviceStats.entries.containsKey(rollupKey)).forEach(rollupKey -> lastUpdateMap.put(rollupKey, (long) serviceStats.entries.get(rollupKey).latestValue));
        getLastRollupTimeFromQuery(currentState, lastUpdateMap);
    }));
}
Also used : Service(com.vmware.xenon.common.Service) ServiceTypeCluster(com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster) QueryTask(com.vmware.xenon.services.common.QueryTask) ServiceDocument(com.vmware.xenon.common.ServiceDocument) AggregationType(com.vmware.xenon.common.ServiceStats.TimeSeriesStats.AggregationType) MatchType(com.vmware.xenon.services.common.QueryTask.QueryTerm.MatchType) Utils(com.vmware.xenon.common.Utils) TaskFactoryService(com.vmware.xenon.services.common.TaskFactoryService) Map(java.util.Map) URI(java.net.URI) Builder(com.vmware.xenon.services.common.QueryTask.Builder) EnumSet(java.util.EnumSet) Collection(java.util.Collection) Set(java.util.Set) Occurance(com.vmware.xenon.services.common.QueryTask.Query.Occurance) ServiceStats(com.vmware.xenon.common.ServiceStats) List(java.util.List) TimeSeriesStats(com.vmware.xenon.common.ServiceStats.TimeSeriesStats) UriUtils(com.vmware.xenon.common.UriUtils) Entry(java.util.Map.Entry) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) TaskState(com.vmware.xenon.common.TaskState) FactoryService(com.vmware.xenon.common.FactoryService) NumericRange(com.vmware.xenon.services.common.QueryTask.NumericRange) TaskService(com.vmware.xenon.services.common.TaskService) SortedMap(java.util.SortedMap) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) TaskUtils(com.vmware.photon.controller.model.tasks.TaskUtils) InMemoryResourceMetric(com.vmware.photon.controller.model.monitoring.InMemoryResourceMetricService.InMemoryResourceMetric) HashMap(java.util.HashMap) PhotonModelUtils(com.vmware.photon.controller.model.resources.util.PhotonModelUtils) ResourceMetricsService(com.vmware.photon.controller.model.monitoring.ResourceMetricsService) ArrayList(java.util.ArrayList) ServiceUriPaths(com.vmware.xenon.services.common.ServiceUriPaths) HashSet(java.util.HashSet) Query(com.vmware.xenon.services.common.QueryTask.Query) UriPaths(com.vmware.photon.controller.model.UriPaths) PropertyUsageOption(com.vmware.xenon.common.ServiceDocumentDescription.PropertyUsageOption) TimeBin(com.vmware.xenon.common.ServiceStats.TimeSeriesStats.TimeBin) OperationSequence(com.vmware.xenon.common.OperationSequence) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) TypeName(com.vmware.xenon.common.ServiceDocumentDescription.TypeName) TaskStage(com.vmware.xenon.common.TaskState.TaskStage) TimeUnit(java.util.concurrent.TimeUnit) TreeMap(java.util.TreeMap) InMemoryResourceMetricService(com.vmware.photon.controller.model.monitoring.InMemoryResourceMetricService) ClusterUtil(com.vmware.photon.controller.model.util.ClusterUtil) ServiceDocumentDescription(com.vmware.xenon.common.ServiceDocumentDescription) Comparator(java.util.Comparator) QuerySpecification(com.vmware.xenon.services.common.QueryTask.QuerySpecification) Collections(java.util.Collections) OperationJoin(com.vmware.xenon.common.OperationJoin) PhotonModelUriUtils.createInventoryUri(com.vmware.photon.controller.model.util.PhotonModelUriUtils.createInventoryUri) ServiceStats(com.vmware.xenon.common.ServiceStats) HashMap(java.util.HashMap) URI(java.net.URI)

Aggregations

ServiceStats (com.vmware.xenon.common.ServiceStats)13 ServiceStat (com.vmware.xenon.common.ServiceStats.ServiceStat)7 ComputeDescription (com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription)6 ComputeState (com.vmware.photon.controller.model.resources.ComputeService.ComputeState)6 ArrayList (java.util.ArrayList)6 Map (java.util.Map)6 ResourceMetrics (com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics)5 ResourcePoolState (com.vmware.photon.controller.model.resources.ResourcePoolService.ResourcePoolState)5 SingleResourceStatsCollectionTaskState (com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState)5 StatsCollectionTaskState (com.vmware.photon.controller.model.tasks.monitoring.StatsCollectionTaskService.StatsCollectionTaskState)5 ServiceDocumentQueryResult (com.vmware.xenon.common.ServiceDocumentQueryResult)5 BaseModelTest (com.vmware.photon.controller.model.helpers.BaseModelTest)4 Test (org.junit.Test)4 ScheduledTaskState (com.vmware.photon.controller.model.tasks.ScheduledTaskService.ScheduledTaskState)3 Operation (com.vmware.xenon.common.Operation)3 UriPaths (com.vmware.photon.controller.model.UriPaths)2 InMemoryResourceMetricService (com.vmware.photon.controller.model.monitoring.InMemoryResourceMetricService)2 InMemoryResourceMetric (com.vmware.photon.controller.model.monitoring.InMemoryResourceMetricService.InMemoryResourceMetric)2 ResourceMetricsService (com.vmware.photon.controller.model.monitoring.ResourceMetricsService)2 QueryUtils (com.vmware.photon.controller.model.query.QueryUtils)2