Examples with QueryTask com.vmware.xenon.services.common.QueryTask used on opensource projects

Example 96 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class StatsCollectionTaskServiceTest method testStatsCollection.

private void testStatsCollection(boolean testOnCluster) throws Throwable {
    VerificationHost metricHost = null;
    if (testOnCluster) {
        metricHost = this.setupMetricHost();
    }
    // Use this.host if metricHost is null.
    VerificationHost verificationHost = (metricHost == null ? this.host : metricHost);
    // create a metric host
    // create a compute description for all the computes
    ComputeDescription cDesc = new ComputeDescription();
    cDesc.name = UUID.randomUUID().toString();
    cDesc.statsAdapterReference = UriUtils.buildUri(this.host, MockStatsAdapter.SELF_LINK);
    ComputeDescription descReturnState = postServiceSynchronously(ComputeDescriptionService.FACTORY_LINK, cDesc, ComputeDescription.class);
    // create multiple computes
    ComputeState computeState = new ComputeState();
    computeState.name = UUID.randomUUID().toString();
    computeState.descriptionLink = descReturnState.documentSelfLink;
    List<String> computeLinks = new ArrayList<>(this.numResources);
    for (int i = 0; i < this.numResources; i++) {
        ComputeState res = postServiceSynchronously(ComputeService.FACTORY_LINK, computeState, ComputeState.class);
        computeLinks.add(res.documentSelfLink);
    }
    // create a resource pool including all the created computes
    ResourcePoolState rpState = new ResourcePoolState();
    rpState.name = UUID.randomUUID().toString();
    rpState.properties = EnumSet.of(ResourcePoolProperty.ELASTIC);
    rpState.query = Query.Builder.create().addKindFieldClause(ComputeState.class).addInClause(ServiceDocument.FIELD_NAME_SELF_LINK, computeLinks).build();
    ResourcePoolState rpReturnState = postServiceSynchronously(ResourcePoolService.FACTORY_LINK, rpState, ResourcePoolState.class);
    // create a stats collection scheduler task
    StatsCollectionTaskState statCollectionState = new StatsCollectionTaskState();
    statCollectionState.resourcePoolLink = rpReturnState.documentSelfLink;
    statCollectionState.options = EnumSet.of(TaskOption.SELF_DELETE_ON_COMPLETION);
    ScheduledTaskState statsCollectionTaskState = new ScheduledTaskState();
    statsCollectionTaskState.factoryLink = StatsCollectionTaskService.FACTORY_LINK;
    statsCollectionTaskState.initialStateJson = Utils.toJson(statCollectionState);
    statsCollectionTaskState.intervalMicros = TimeUnit.SECONDS.toMicros(2);
    statsCollectionTaskState = postServiceSynchronously(ScheduledTaskService.FACTORY_LINK, statsCollectionTaskState, ScheduledTaskState.class);
    ServiceDocumentQueryResult res = this.host.getFactoryState(UriUtils.buildExpandLinksQueryUri(UriUtils.buildUri(this.host, ScheduledTaskService.FACTORY_LINK)));
    assertTrue(res.documents.size() > 0);
    // the last successful collection time should be populated as an in memory stat.
    for (int i = 0; i < this.numResources; i++) {
        String statsUriPath = UriUtils.buildUriPath(computeLinks.get(i), ServiceHost.SERVICE_URI_SUFFIX_STATS);
        this.host.waitFor("Error waiting for in memory stats", () -> {
            ServiceStats resStats = getServiceSynchronously(statsUriPath, ServiceStats.class);
            boolean returnStatus = false;
            for (ServiceStat stat : resStats.entries.values()) {
                if (stat.latestValue > 0) {
                    returnStatus = true;
                    break;
                }
            }
            return returnStatus;
        });
    }
    host.log(Level.INFO, "Successfully verified that all the last collection time is available in memory.");
    // persisted at a per metric level along with the last collection run time
    for (String computeLink : computeLinks) {
        ResourceMetrics metric = getResourceMetrics(verificationHost, computeLink, MockStatsAdapter.KEY_1);
        assertNotNull("The resource metric for" + MockStatsAdapter.KEY_1 + " should not be null ", metric);
        assertEquals(metric.entries.size(), 1);
        assertEquals(metric.customProperties.get("prop1"), "val1");
        ResourceMetrics metric2 = getResourceMetrics(verificationHost, computeLink, MockStatsAdapter.KEY_2);
        assertNotNull("The resource metric for" + MockStatsAdapter.KEY_2 + "should not be null ", metric2);
        assertEquals(metric2.entries.size(), 1);
        String lastSuccessfulRunMetricKey = UriUtils.getLastPathSegment(MockStatsAdapter.SELF_LINK) + StatsUtil.SEPARATOR + PhotonModelConstants.LAST_SUCCESSFUL_STATS_COLLECTION_TIME;
        ResourceMetrics metricLastRun = getResourceMetrics(verificationHost, computeLink, lastSuccessfulRunMetricKey);
        assertNotNull("The resource metric for" + lastSuccessfulRunMetricKey + " should not be null ", metricLastRun);
    }
    host.log(Level.INFO, "Successfully verified that the required resource metrics are persisted in the resource metrics table");
    // Verify sorted order of the metrics versions by timestamp
    for (String computeLink : computeLinks) {
        // get all versions
        QueryTask qt = QueryTask.Builder.createDirectTask().addOption(QueryOption.EXPAND_CONTENT).addOption(QueryOption.SORT).orderAscending(ServiceDocument.FIELD_NAME_SELF_LINK, TypeName.STRING).setQuery(Query.Builder.create().addKindFieldClause(ResourceMetrics.class).addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, UriUtils.buildUriPath(ResourceMetricsService.FACTORY_LINK, UriUtils.getLastPathSegment(computeLink)), MatchType.PREFIX).addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, MockStatsAdapter.KEY_1), NumericRange.createDoubleRange(Double.MIN_VALUE, Double.MAX_VALUE, true, true)).build()).build();
        verificationHost.createQueryTaskService(qt, false, true, qt, null);
        ResourceMetrics prevMetric = null;
        for (String documentLink : qt.results.documentLinks) {
            ResourceMetrics metric = Utils.fromJson(qt.results.documents.get(documentLink), ResourceMetrics.class);
            if (prevMetric == null) {
                prevMetric = metric;
                continue;
            }
            assertTrue(prevMetric.timestampMicrosUtc < metric.timestampMicrosUtc);
        }
    }
    // verify that the aggregation tasks have been deleted
    this.host.waitFor("Timeout waiting for task to expire", () -> {
        ServiceDocumentQueryResult collectRes = this.host.getFactoryState(UriUtils.buildUri(this.host, StatsCollectionTaskService.FACTORY_LINK));
        if (collectRes.documentLinks.size() == 0) {
            return true;
        }
        return false;
    });
    if (testOnCluster) {
        this.cleanUpMetricHost(metricHost);
    }
}

Example 97 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class QueryUtils method createAccountQuery.

/**
 * Create a QueryTask to fetch an endpoint based on account ID for the given account type.
 */
public static QueryTask createAccountQuery(String accountId, String accountType, List<String> tenantLinks) {
    Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(ComputeState.class).addFieldClause(ComputeState.FIELD_NAME_TYPE, ComputeType.ENDPOINT_HOST).addCompositeFieldClause(ComputeState.FIELD_NAME_CUSTOM_PROPERTIES, "__endpointType", accountType).addCompositeFieldClause(ComputeState.FIELD_NAME_CUSTOM_PROPERTIES, PhotonModelConstants.CLOUD_ACCOUNT_ID, accountId).addCompositeFieldClause(ComputeState.FIELD_NAME_CUSTOM_PROPERTIES, PhotonModelConstants.AUTO_DISCOVERED_ENTITY, Boolean.TRUE.toString(), Occurance.MUST_NOT_OCCUR).addFieldClause("endpointLinks.item", "*", MatchType.WILDCARD, Occurance.MUST_OCCUR);
    // it must match all tenantLinks in order to determine whether a compute host already
    // exists.
    addTenantLinks(qBuilder, tenantLinks);
    QueryTask queryTask = QueryTask.Builder.createDirectTask().setQuery(qBuilder.build()).addOption(QueryOption.EXPAND_CONTENT).addOption(QueryOption.TOP_RESULTS).setResultLimit(100).build();
    return queryTask;
}

Example 98 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method disassociateIfNotAttachedToCompute.

private DeferredResult<Operation> disassociateIfNotAttachedToCompute(StorageEnumContext context, DiskState diskState) {
    Query query = Query.Builder.create().addKindFieldClause(ComputeService.ComputeState.class).addCollectionItemClause(ComputeService.ComputeState.FIELD_NAME_DISK_LINKS, diskState.documentSelfLink).build();
    QueryTask queryTask = QueryTask.Builder.createDirectTask().addOption(QueryOption.COUNT).addOption(QueryOption.INDEXED_METADATA).setQuery(query).build();
    queryTask.tenantLinks = diskState.tenantLinks;
    if (queryTask.documentExpirationTimeMicros == 0) {
        queryTask.documentExpirationTimeMicros = Utils.getNowMicrosUtc() + QueryUtils.TEN_MINUTES_IN_MICROS;
    }
    return QueryUtils.startInventoryQueryTask(this, queryTask).thenCompose(result -> {
        if (result.results != null && result.results.documentCount != 0) {
            logFine(() -> String.format("Won't disassociate disk state %s, as it is attached to machine", diskState.documentSelfLink));
            return DeferredResult.completed(new Operation());
        }
        logFine(() -> String.format("Disassociating disk state %s", diskState.documentSelfLink));
        Operation operation = PhotonModelUtils.createRemoveEndpointLinksOperation(this, context.request.endpointLink, diskState);
        if (operation == null) {
            return DeferredResult.completed(new Operation());
        }
        CompletionHandler completion = operation.getCompletion();
        return sendWithDeferredResult(operation).whenComplete(completion::handle).whenComplete((o, e) -> {
            final String message = "Disassociate disk state stale %s state";
            if (e != null) {
                logWarning(message + ": ERROR - %s", diskState.documentSelfLink, Utils.toString(e));
            } else {
                logFine(message + ": SUCCESS", diskState.documentSelfLink);
            }
        });
    });
}

Example 99 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class AzureTestUtil method getResourceMetrics.

/**
 * Query to get ResourceMetrics document for a specific resource containing a specific metric.
 *
 * @param host
 *            host against which query is triggered
 * @param resourceLink
 *            Link to the resource on which stats are being collected.
 * @param metricKey
 *            Metric name.
 * @return ResourceMetrics document.
 */
public static ResourceMetrics getResourceMetrics(VerificationHost host, String resourceLink, String metricKey) {
    QueryTask qt = QueryTask.Builder.createDirectTask().addOption(QueryOption.EXPAND_CONTENT).addOption(QueryOption.SORT).orderDescending(ServiceDocument.FIELD_NAME_SELF_LINK, ServiceDocumentDescription.TypeName.STRING).setQuery(QueryTask.Query.Builder.create().addKindFieldClause(ResourceMetrics.class).addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, UriUtils.buildUriPath(ResourceMetricsService.FACTORY_LINK, UriUtils.getLastPathSegment(resourceLink)), QueryTerm.MatchType.PREFIX).addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, metricKey), QueryTask.NumericRange.createDoubleRange(0.0, Double.MAX_VALUE, true, true)).build()).build();
    Operation op = QueryUtils.createQueryTaskOperation(host, qt, ClusterUtil.ServiceTypeCluster.METRIC_SERVICE).setReferer(host.getUri()).setBody(qt).setCompletion((o, e) -> {
        if (e != null) {
            host.log(Level.INFO, e.toString());
        }
    });
    Operation result = host.waitForResponse(op);
    QueryTask qtResult = result.getBody(QueryTask.class);
    ResourceMetrics resourceMetric = null;
    if (qtResult.results.documentLinks.size() > 0) {
        String documentLink = qtResult.results.documentLinks.get(0);
        resourceMetric = Utils.fromJson(qtResult.results.documents.get(documentLink), ResourceMetrics.class);
    }
    return resourceMetric;
}

Example 100 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class LongRunEndToEndAzureStatsAggregation method verifyTimeBinMetrics.

/**
 * Performs check to verify time bin metrics are available for every compute resource and also
 * checks the occurrence of estimated charges for one of the resource metrics.
 */
private void verifyTimeBinMetrics(ServiceDocumentQueryResult res) {
    List<Object> results = new ArrayList<>();
    for (String computeResourceLink : res.documentLinks) {
        for (String metricName : AzureTestUtil.getMetricNames()) {
            QuerySpecification querySpec = new QuerySpecification();
            querySpec.query = QueryTask.Query.Builder.create().addKindFieldClause(ResourceMetrics.class).addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, UriUtils.buildUriPath(ResourceMetricsService.FACTORY_LINK, computeResourceLink), QueryTerm.MatchType.PREFIX).addRangeClause(QuerySpecification.buildCompositeFieldName(ResourceMetrics.FIELD_NAME_ENTRIES, metricName), NumericRange.createDoubleRange(0.0, Double.MAX_VALUE, true, true)).build();
            QueryTask qt = QueryTask.Builder.createDirectTask().addOption(QueryOption.TOP_RESULTS).addOption(QueryOption.INCLUDE_ALL_VERSIONS).setResultLimit(1).addOption(QueryOption.EXPAND_CONTENT).addOption(QueryOption.SORT).orderDescending(ServiceDocument.FIELD_NAME_SELF_LINK, ServiceDocumentDescription.TypeName.STRING).setQuery(querySpec.query).build();
            this.host.createQueryTaskService(qt, false, true, qt, null);
            if (qt.results.documentLinks.size() > 0 && qt.results.documentLinks.get(0) != null) {
                results.add(qt.results.documents.get(qt.results.documentLinks.get(0)));
            }
        }
    }
    long expectedExpirationTime = Utils.getNowMicrosUtc() + TimeUnit.DAYS.toMicros(DEFAULT_RETENTION_LIMIT_DAYS);
    for (Object aggrDocument : results) {
        ResourceMetrics aggrMetric = Utils.fromJson(aggrDocument, ResourceMetrics.class);
        // Make sure all the documents have expiration time set.
        assertTrue("Expiration time is not correctly set.", aggrMetric.documentExpirationTimeMicros < expectedExpirationTime);
        // The assertion here checks whether we are aggregating only on latest value. To
        // that effect, here is the breakdown for the check:
        // count = num of resources: one value for each resource
        // sum = null: not specified in the aggregate type set
        assertNotNull("Value is not set", aggrMetric.entries.get(0));
    }
}