Search in sources :

Example 41 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class MetricsClusterTest method getDocumentCount.

private Long getDocumentCount(VerificationHost host, String factoryLink) {
    QueryTask queryTask = QueryTask.Builder.createDirectTask().addOption(QueryOption.INCLUDE_ALL_VERSIONS).addOption(QueryOption.COUNT).setQuery(Query.Builder.create().addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, factoryLink, MatchType.PREFIX).build()).build();
    host.createQueryTaskService(queryTask, false, true, queryTask, null);
    return queryTask.results.documentCount;
}
Also used : QueryTask(com.vmware.xenon.services.common.QueryTask)

Example 42 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class AWSCostStatsService method createQueryForComputeStatesByAccount.

/**
 * Method creates a query operation to get the compute states corresponding to the specified account ID.
 * The list of the resultant compute states is then passed to the specified handler for processing.
 *
 * @param context
 * @param accountId
 * @param queryResultConsumer
 * @return operation object representing the query
 */
protected Operation createQueryForComputeStatesByAccount(AWSCostStatsCreationContext context, String accountId, Consumer<List<ComputeState>> queryResultConsumer) {
    Query awsAccountsQuery = Query.Builder.create().addKindFieldClause(ComputeState.class).addFieldClause(ComputeState.FIELD_NAME_TYPE, ComputeType.ENDPOINT_HOST).addCompositeFieldClause(ComputeState.FIELD_NAME_CUSTOM_PROPERTIES, EndpointAllocationTaskService.CUSTOM_PROP_ENPOINT_TYPE, PhotonModelConstants.EndpointType.aws.name()).addCompositeFieldClause(ComputeState.FIELD_NAME_CUSTOM_PROPERTIES, AWS_ACCOUNT_ID_KEY, accountId).addInCollectionItemClause(ComputeState.FIELD_NAME_TENANT_LINKS, context.computeDesc.tenantLinks).build();
    QueryTask queryTask = QueryTask.Builder.createDirectTask().addOption(QueryOption.EXPAND_CONTENT).setQuery(awsAccountsQuery).build();
    queryTask.setDirect(true);
    queryTask.tenantLinks = context.computeDesc.tenantLinks;
    return QueryUtils.createQueryTaskOperation(this, queryTask, ServiceTypeCluster.INVENTORY_SERVICE).setCompletion((o, e) -> {
        if (e != null) {
            getFailureConsumer(context).accept(e);
            return;
        }
        QueryTask responseTask = o.getBody(QueryTask.class);
        List<ComputeState> accountComputeStates = responseTask.results.documents.values().stream().map(s -> Utils.fromJson(s, ComputeState.class)).filter(cs -> cs.parentLink == null && cs.endpointLink != null && !CollectionUtils.isEmpty(cs.endpointLinks)).collect(Collectors.toList());
        queryResultConsumer.accept(accountComputeStates);
    });
}
Also used : AuthCredentialsServiceState(com.vmware.xenon.services.common.AuthCredentialsService.AuthCredentialsServiceState) Arrays(java.util.Arrays) DateTimeZone(org.joda.time.DateTimeZone) AWS_ACCOUNT_BILL_PROCESSED_TIME_MILLIS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_ACCOUNT_BILL_PROCESSED_TIME_MILLIS) AWSCsvBillParser(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSCsvBillParser) ServiceTypeCluster(com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster) QueryTask(com.vmware.xenon.services.common.QueryTask) ServiceDocument(com.vmware.xenon.common.ServiceDocument) ComputeType(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription.ComputeType) EndpointAllocationTaskService(com.vmware.photon.controller.model.tasks.EndpointAllocationTaskService) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState) ProgressListener(com.amazonaws.event.ProgressListener) STORAGE_TYPE_EBS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_EBS) Utils(com.vmware.xenon.common.Utils) STORAGE_TYPE_S3(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_S3) ACCOUNT_IS_AUTO_DISCOVERED(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.ACCOUNT_IS_AUTO_DISCOVERED) Map(java.util.Map) URI(java.net.URI) ProgressEventType(com.amazonaws.event.ProgressEventType) AWSMissingResourcesEnumerationService(com.vmware.photon.controller.model.adapters.awsadapter.enumeration.AWSMissingResourcesEnumerationService) Path(java.nio.file.Path) AWSClientManager(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManager) EndpointState(com.vmware.photon.controller.model.resources.EndpointService.EndpointState) PrintWriter(java.io.PrintWriter) ComputeStatsResponse(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse) ComputeStatsRequest(com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest) StatelessService(com.vmware.xenon.common.StatelessService) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) TaskManager(com.vmware.photon.controller.model.adapters.util.TaskManager) Set(java.util.Set) DiskState(com.vmware.photon.controller.model.resources.DiskService.DiskState) Occurance(com.vmware.xenon.services.common.QueryTask.Query.Occurance) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) AwsServiceDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsServiceDetailDto) Objects(java.util.Objects) List(java.util.List) AWS_LINKED_ACCOUNT_IDS(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_LINKED_ACCOUNT_IDS) Stream(java.util.stream.Stream) UriUtils(com.vmware.xenon.common.UriUtils) Entry(java.util.Map.Entry) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) AwsServices(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSCsvBillParser.AwsServices) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) SingleResourceTaskCollectionStage(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceTaskCollectionStage) AwsAccountDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsAccountDetailDto) TransferManager(com.amazonaws.services.s3.transfer.TransferManager) QueryByPages(com.vmware.photon.controller.model.query.QueryUtils.QueryByPages) OperationContext(com.vmware.xenon.common.OperationContext) AWS_ACCOUNT_ID_KEY(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWS_ACCOUNT_ID_KEY) HashMap(java.util.HashMap) ComputeDescriptionService(com.vmware.photon.controller.model.resources.ComputeDescriptionService) GetObjectRequest(com.amazonaws.services.s3.model.GetObjectRequest) ResourceMetricsService(com.vmware.photon.controller.model.monitoring.ResourceMetricsService) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) ProgressEvent(com.amazonaws.event.ProgressEvent) HashSet(java.util.HashSet) AuthCredentialsService(com.vmware.xenon.services.common.AuthCredentialsService) Query(com.vmware.xenon.services.common.QueryTask.Query) UriPaths(com.vmware.photon.controller.model.UriPaths) CollectionUtils(org.apache.commons.collections.CollectionUtils) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) BiConsumer(java.util.function.BiConsumer) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) AwsClientType(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AwsClientType) StatsUtil(com.vmware.photon.controller.model.tasks.monitoring.StatsUtil) ExecutorService(java.util.concurrent.ExecutorService) AdapterUtils(com.vmware.photon.controller.model.adapters.util.AdapterUtils) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) ResourceState(com.vmware.photon.controller.model.resources.ResourceState) Files(java.nio.file.Files) StringWriter(java.io.StringWriter) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) IOException(java.io.IOException) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) AwsResourceDetailDto(com.vmware.photon.controller.model.adapters.aws.dto.AwsResourceDetailDto) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) LocalDate(org.joda.time.LocalDate) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) Paths(java.nio.file.Paths) ClusterUtil(com.vmware.photon.controller.model.util.ClusterUtil) AWSStatsNormalizer(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSStatsNormalizer) ComputeStateWithDescription(com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription) PhotonModelConstants(com.vmware.photon.controller.model.constants.PhotonModelConstants) AWSClientManagerFactory(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManagerFactory) ServiceDocumentDescription(com.vmware.xenon.common.ServiceDocumentDescription) Collections(java.util.Collections) OperationJoin(com.vmware.xenon.common.OperationJoin) PhotonModelUriUtils.createInventoryUri(com.vmware.photon.controller.model.util.PhotonModelUriUtils.createInventoryUri) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) QueryTask(com.vmware.xenon.services.common.QueryTask) Query(com.vmware.xenon.services.common.QueryTask.Query)

Example 43 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class AWSCostStatsService method getQueryTaskForMetric.

private QueryTask getQueryTaskForMetric(ComputeState accountComputeState) {
    Query.Builder builder = Query.Builder.create();
    builder.addKindFieldClause(ResourceMetrics.class);
    builder.addCompositeFieldClause(ResourceMetrics.FIELD_NAME_CUSTOM_PROPERTIES, ResourceMetrics.PROPERTY_RESOURCE_LINK, accountComputeState.documentSelfLink);
    builder.addCompositeFieldClause(ResourceMetrics.FIELD_NAME_CUSTOM_PROPERTIES, PhotonModelConstants.CONTAINS_BILL_PROCESSED_TIME_STAT, Boolean.TRUE.toString());
    QueryTask.Builder qTaskBuilder = QueryTask.Builder.createDirectTask().addOption(QueryOption.SORT).addOption(QueryOption.TOP_RESULTS).addOption(QueryOption.INCLUDE_ALL_VERSIONS).addOption(QueryOption.EXPAND_CONTENT).orderDescending(ServiceDocument.FIELD_NAME_SELF_LINK, ServiceDocumentDescription.TypeName.STRING).setResultLimit(1).setQuery(builder.build());
    if (!AWSConstants.SHOULD_REFRESH_INDEX) {
        qTaskBuilder.addOption(QueryOption.DO_NOT_REFRESH);
    }
    QueryTask qTask = qTaskBuilder.build();
    qTask.documentExpirationTimeMicros = Utils.fromNowMicrosUtc(QueryUtils.TEN_MINUTES_IN_MICROS);
    return qTask;
}
Also used : QueryTask(com.vmware.xenon.services.common.QueryTask) Query(com.vmware.xenon.services.common.QueryTask.Query)

Example 44 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class AWSNetworkStateEnumerationAdapterService method getLocalSubnetStates.

/**
 * Gets the Subnet information from the local database to perform updates to existing subnet
 * states.
 */
private void getLocalSubnetStates(AWSNetworkStateCreationContext context, AWSNetworkStateCreationStage next) {
    if (context.subnets.isEmpty()) {
        handleNetworkStateChanges(context, next);
        return;
    }
    QueryTask q = createQueryToGetExistingSubnetStatesFilteredByDiscoveredSubnets(context.subnets.keySet(), context.request.parentComputeLink, context.request.request.endpointLink, context.request.regionId, context.request.tenantLinks);
    // create the query to find resources
    QueryUtils.startInventoryQueryTask(this, q).whenComplete((queryTask, e) -> {
        if (e != null) {
            logSevere(() -> String.format("Failed retrieving query results: %s", e.toString()));
            finishWithFailure(context, e);
            return;
        }
        if (queryTask.results.documents != null) {
            for (Object s : queryTask.results.documents.values()) {
                SubnetState subnetState = Utils.fromJson(s, SubnetState.class);
                context.localSubnetStateMap.put(subnetState.id, subnetState);
            }
        }
        logFine(() -> String.format("%d subnet states found.", queryTask.results.documentCount));
        handleNetworkStateChanges(context, next);
    });
}
Also used : QueryTask(com.vmware.xenon.services.common.QueryTask) SubnetState(com.vmware.photon.controller.model.resources.SubnetService.SubnetState) AWSNetworkUtils.mapSubnetToSubnetState(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSNetworkUtils.mapSubnetToSubnetState)

Example 45 with QueryTask

use of com.vmware.xenon.services.common.QueryTask in project photon-model by vmware.

the class AWSEnumerationUtils method getCDsRepresentingVMsInLocalSystemCreatedByEnumerationQuery.

/**
 * Get all the compute descriptions already in the system that correspond to virtual machine and filter by.
 * This query is primarily used during instance discovery to find compute descriptions that exist in the system
 * to match the instances received from AWS.
 * The query filters out compute descriptions that represent compute hosts and also checks for other conditions
 * as below:
 * - Environment name(AWS),
 * - id (instance type),
 * - ZoneId(placement).
 * - Created from the enumeration task.
 * Compute hosts are modeled to support VM guests.So excluding them from the query to get
 * compute descriptions for VMs.
 */
public static QueryTask getCDsRepresentingVMsInLocalSystemCreatedByEnumerationQuery(Set<InstanceDescKey> descriptionsSet, List<String> tenantLinks, String regionId, String parentComputeLink, String endpointLink) {
    String sourceTaskName = QueryTask.QuerySpecification.buildCompositeFieldName(ComputeState.FIELD_NAME_CUSTOM_PROPERTIES, SOURCE_TASK_LINK);
    Query query = Query.Builder.create().addKindFieldClause(ComputeDescription.class).addFieldClause(ComputeDescription.FIELD_NAME_ENVIRONMENT_NAME, ComputeDescription.ENVIRONMENT_NAME_AWS).addFieldClause(ComputeDescription.FIELD_NAME_REGION_ID, regionId).addFieldClause(ComputeDescription.FIELD_NAME_COMPUTE_HOST_LINK, parentComputeLink).addFieldClause(sourceTaskName, ResourceEnumerationTaskService.FACTORY_LINK).build();
    // Instance type and zone should fall in one of the passed in values
    Query groupFilter = new Query();
    groupFilter.occurance = Occurance.MUST_OCCUR;
    for (InstanceDescKey key : descriptionsSet) {
        Query itf = new Query().setTermPropertyName(ComputeDescription.FIELD_NAME_ID).setTermMatchValue(key.instanceType);
        itf.occurance = Occurance.MUST_OCCUR;
        Query zf = new Query().setTermPropertyName(ComputeDescription.FIELD_NAME_ZONE_ID).setTermMatchValue(key.zoneId);
        zf.occurance = Occurance.MUST_OCCUR;
        Query d = new Query();
        d.occurance = Occurance.SHOULD_OCCUR;
        d.addBooleanClause(itf);
        d.addBooleanClause(zf);
        groupFilter.addBooleanClause(d);
    }
    query.addBooleanClause(groupFilter);
    QueryTask queryTask = QueryTask.Builder.createDirectTask().setQuery(query).addOption(QueryOption.EXPAND_CONTENT).addOption(QueryOption.INDEXED_METADATA).addOption(QueryOption.TOP_RESULTS).setResultLimit(getQueryResultLimit()).build();
    queryTask.documentSelfLink = UUID.randomUUID().toString();
    queryTask.tenantLinks = tenantLinks;
    return queryTask;
}
Also used : QueryTask(com.vmware.xenon.services.common.QueryTask) Query(com.vmware.xenon.services.common.QueryTask.Query) ComputeDescription(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription)

Aggregations

QueryTask (com.vmware.xenon.services.common.QueryTask)147 Query (com.vmware.xenon.services.common.QueryTask.Query)62 Operation (com.vmware.xenon.common.Operation)61 ComputeState (com.vmware.photon.controller.model.resources.ComputeService.ComputeState)39 ArrayList (java.util.ArrayList)26 QueryUtils (com.vmware.photon.controller.model.query.QueryUtils)20 List (java.util.List)20 ResourceMetrics (com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics)19 UriUtils (com.vmware.xenon.common.UriUtils)18 Utils (com.vmware.xenon.common.Utils)17 HashSet (java.util.HashSet)16 HashMap (java.util.HashMap)14 QueryOption (com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption)12 TimeUnit (java.util.concurrent.TimeUnit)12 ComputeEnumerateResourceRequest (com.vmware.photon.controller.model.adapterapi.ComputeEnumerateResourceRequest)11 AdapterUtils (com.vmware.photon.controller.model.adapters.util.AdapterUtils)11 ComputeDescription (com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription)11 ServiceTypeCluster (com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster)11 URI (java.net.URI)11 Set (java.util.Set)11