Examples with QueryByPages com.vmware.photon.controller.model.query.QueryUtils.QueryByPages used on opensource projects

Example 11 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method disassociateStorageDescription.

/*
     * Disassociate local storage accounts and all resources inside them that no longer exist in
     * Azure
     *
     * The logic works by recording a timestamp when enumeration starts. This timestamp is used to
     * lookup resources which haven't been touched as part of current enumeration cycle. The other
     * data point this method uses is the storage accounts discovered as part of get storage
     * accounts call.
     *
     * A disassociate on a storage description is invoked only if it meets two criteria: -
     * Timestamp older
     * than current enumeration cycle. - Storage account is not present on Azure.
     */
private void disassociateStorageDescription(StorageEnumContext context, StorageEnumStages next) {
    Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(StorageDescription.class).addRangeClause(StorageDescription.FIELD_NAME_UPDATE_TIME_MICROS, QueryTask.NumericRange.createLessThanRange(context.enumerationStartTimeInMicros));
    QueryByPages<StorageDescription> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), StorageDescription.class, context.parentCompute.tenantLinks, context.request.endpointLink, context.parentCompute.documentSelfLink).setMaxPageSize(QueryUtils.MAX_RESULT_LIMIT).setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
    List<DeferredResult<Operation>> disassociateDRs = new ArrayList<>();
    queryLocalStates.queryDocuments(sd -> {
        if (context.storageAccountIds.contains(sd.id)) {
            return;
        }
        // Deleting the localResourceState is done by disassociating the
        // endpointLink from the localResourceState. If the localResourceState
        // isn't associated with any other endpointLink, it should be eventually
        // deleted by the groomer task
        Operation disassociateOp = PhotonModelUtils.createRemoveEndpointLinksOperation(this, context.request.endpointLink, sd);
        if (disassociateOp == null) {
            return;
        }
        // NOTE: The original Op is set with completion that must be executed.
        // Since sendWithDeferredResult is used we must manually call it, otherwise it's
        // just ignored.
        CompletionHandler disassociateOpCompletion = disassociateOp.getCompletion();
        DeferredResult<Operation> disassociateDR = sendWithDeferredResult(disassociateOp).whenComplete(disassociateOpCompletion::handle).whenComplete((o, e) -> {
            final String message = "Disassociate stale %s state";
            if (e != null) {
                logWarning(message + ": FAILED with %s", sd.documentSelfLink, Utils.toString(e));
            } else {
                log(Level.FINEST, message + ": SUCCESS", sd.documentSelfLink);
            }
        });
        disassociateDRs.add(disassociateDR);
    }).thenCompose(ignore -> DeferredResult.allOf(disassociateDRs)).whenComplete((r, e) -> {
        logFine(() -> "Finished disassociation of storage descriptions for Azure");
        context.subStage = next;
        handleSubStage(context);
    });
}

Example 12 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method disassociateResourceGroupStates.

/*
     * Disassociate local storage containers that no longer exist in Azure
     *
     * The logic works by recording a timestamp when enumeration starts. This timestamp is used to
     * lookup resources which haven't been touched as part of current enumeration cycle. The other
     * data point this method uses is the storage accounts discovered as part of get storage
     * accounts call.
     *
     * A disassociate on a resource group state is invoked only if it meets two criteria: -
     * Timestamp
     * older than current enumeration cycle. - Storage container is not present on Azure.
     */
private void disassociateResourceGroupStates(StorageEnumContext context, StorageEnumStages next) {
    Query.Builder qBuilder = newResourceGroupQueryBuilder(context).addRangeClause(ResourceGroupState.FIELD_NAME_UPDATE_TIME_MICROS, QueryTask.NumericRange.createLessThanRange(context.enumerationStartTimeInMicros));
    QueryByPages<ResourceGroupState> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), ResourceGroupState.class, context.parentCompute.tenantLinks, null, /* endpointLink */
    context.parentCompute.documentSelfLink).setMaxPageSize(QueryUtils.MAX_RESULT_LIMIT).setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
    List<DeferredResult<Operation>> ops = new ArrayList<>();
    queryLocalStates.queryDocuments(rg -> {
        // repository it means nothing has changed about it.
        if (context.containerIds.contains(canonizeId(rg.id))) {
            return;
        }
        // Deleting the localResourceState is done by disassociating the
        // endpointLink from the localResourceState. If the localResourceState
        // isn't associated with any other endpointLink, it should be eventually
        // deleted by the groomer task
        Operation disassociateOp = PhotonModelUtils.createRemoveEndpointLinksOperation(this, context.request.endpointLink, rg);
        if (disassociateOp == null) {
            return;
        }
        // NOTE: The original Op is set with completion that must be executed.
        // Since sendWithDeferredResult is used we must manually call it, otherwise it's
        // just ignored.
        CompletionHandler disassociateOpCompletion = disassociateOp.getCompletion();
        DeferredResult<Operation> disassociateDR = sendWithDeferredResult(disassociateOp).whenComplete(disassociateOpCompletion::handle).whenComplete((o, e) -> {
            final String message = "Disassociate stale %s state";
            if (e != null) {
                logWarning(message + ": FAILED with %s", rg.documentSelfLink, Utils.toString(e));
            } else {
                log(Level.FINEST, message + ": SUCCESS", rg.documentSelfLink);
            }
        });
        ops.add(disassociateDR);
    }).thenCompose(r -> DeferredResult.allOf(ops)).whenComplete((r, e) -> {
        logFine(() -> "Finished disassociation of storage containers for Azure");
        context.subStage = next;
        handleSubStage(context);
    });
}

Example 13 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method disassociateDiskStates.

/*
     * Disassociate local disk states that no longer exist in Azure
     *
     * The logic works by recording a timestamp when enumeration starts. This timestamp is used to
     * lookup resources which haven't been touched as part of current enumeration cycle. The other
     * data point this method uses is the blob discovered as part of get blob call.
     *
     * A disassociate on a disk state is invoked only if it meets two criteria: - Timestamp older
     * than current enumeration cycle. - blob is not present on Azure.
     */
private void disassociateDiskStates(StorageEnumContext context, StorageEnumStages next) {
    Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(DiskState.class).addFieldClause(DiskState.FIELD_NAME_COMPUTE_HOST_LINK, context.parentCompute.documentSelfLink).addRangeClause(DiskState.FIELD_NAME_UPDATE_TIME_MICROS, QueryTask.NumericRange.createLessThanRange(context.enumerationStartTimeInMicros));
    Query.Builder typeFilterQuery = Query.Builder.create(Occurance.MUST_OCCUR);
    Query blobFilter = Query.Builder.create(Occurance.SHOULD_OCCUR).addFieldClause(AZURE_STORAGE_TYPE, AZURE_STORAGE_BLOBS).build();
    QueryTask.Query diskFilter = QueryTask.Query.Builder.create(QueryTask.Query.Occurance.SHOULD_OCCUR).addFieldClause(AZURE_STORAGE_TYPE, AZURE_STORAGE_DISKS).build();
    typeFilterQuery.addClause(blobFilter);
    typeFilterQuery.addClause(diskFilter);
    qBuilder.addClause(typeFilterQuery.build());
    QueryByPages<DiskState> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), DiskState.class, context.parentCompute.tenantLinks, context.request.endpointLink).setMaxPageSize(getQueryResultLimit());
    queryLocalStates.setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
    List<DeferredResult<Operation>> ops = new ArrayList<>();
    queryLocalStates.queryDocuments(ds -> {
        if (context.blobIds.contains(ds.id)) {
            return;
        }
        ops.add(disassociateIfNotAttachedToCompute(context, ds));
    }).thenCompose(r -> DeferredResult.allOf(ops)).whenComplete((r, e) -> {
        logFine(() -> "Finished disassociation of disk states for Azure");
        context.subStage = next;
        handleSubStage(context);
    });
}

Example 14 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class AWSNetworkStateEnumerationAdapterService method deleteStaleLocalStates.

private DeferredResult<AWSNetworkStateCreationContext> deleteStaleLocalStates(AWSNetworkStateCreationContext context, Class<? extends ResourceState> localStateClass, Set<String> remoteResourcesKeys) {
    final String msg = "Delete %ss that no longer exist in the endpoint: %s";
    logFine(() -> String.format(msg, localStateClass.getSimpleName(), "STARTING"));
    Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(localStateClass).addFieldClause("lifecycleState", LifecycleState.PROVISIONING.toString(), Occurance.MUST_NOT_OCCUR).addRangeClause(ServiceDocument.FIELD_NAME_UPDATE_TIME_MICROS, createLessThanRange(context.enumStartTimeInMicros));
    if (context.request.regionId != null) {
        // Delete resources only in this End-point region
        qBuilder.addFieldClause(ResourceState.FIELD_NAME_REGION_ID, context.request.regionId);
    }
    if (!remoteResourcesKeys.isEmpty() && remoteResourcesKeys.size() <= MAX_RESOURCES_TO_QUERY_ON_DELETE) {
        // do not load resources from enumExternalResourcesIds
        qBuilder.addInClause(ResourceState.FIELD_NAME_ID, remoteResourcesKeys, Occurance.MUST_NOT_OCCUR);
    }
    QueryByPages<? extends ResourceState> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), localStateClass, context.request.tenantLinks, context.request.request.endpointLink);
    queryLocalStates.setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
    List<DeferredResult<Operation>> disassociateDRs = new ArrayList<>();
    // Delete stale resources.
    return queryLocalStates.queryDocuments(localState -> {
        if (remoteResourcesKeys.contains(localState.id)) {
            return;
        }
        // Deleting the localResourceState is done by disassociating the
        // endpointLink from the localResourceState. If the localResourceState
        // isn't associated with any other endpointLink, it should be eventually
        // deleted by the groomer task
        Operation disassociateOp = PhotonModelUtils.createRemoveEndpointLinksOperation(this, context.request.request.endpointLink, localState);
        if (disassociateOp == null) {
            return;
        }
        // NOTE: The original Op is set with completion that must be executed.
        // Since sendWithDeferredResult is used we must manually call it, otherwise it's
        // just ignored.
        CompletionHandler disassociateOpCompletion = disassociateOp.getCompletion();
        DeferredResult<Operation> disassociateDR = sendWithDeferredResult(disassociateOp).whenComplete(disassociateOpCompletion::handle).whenComplete((o, e) -> {
            final String message = "Disassociate stale %s state";
            if (e != null) {
                logWarning(message + ": FAILED with %s", localState.documentSelfLink, Utils.toString(e));
            } else {
                log(Level.FINEST, message + ": SUCCESS", localState.documentSelfLink);
            }
        });
        disassociateDRs.add(disassociateDR);
    }).thenCompose(ignore -> DeferredResult.allOf(disassociateDRs)).thenApply(ignore -> context);
}

Example 15 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class TestUtils method getSubnetStates.

/**
 * Get all SubnetStates within passed NetworkState. In other words, get all subnet states that
 * refer the network state passed.
 */
public static List<SubnetState> getSubnetStates(VerificationHost host, NetworkState networkState) throws Throwable {
    Query queryForReferrers = QueryUtils.queryForReferrers(networkState.documentSelfLink, SubnetState.class, SubnetState.FIELD_NAME_NETWORK_LINK);
    QueryByPages<SubnetState> querySubnetStatesReferrers = new QueryByPages<>(host, queryForReferrers, SubnetState.class, networkState.tenantLinks, null, networkState.computeHostLink);
    DeferredResult<List<SubnetState>> subnetDR = querySubnetStatesReferrers.collectDocuments(Collectors.toList());
    return waitToComplete(subnetDR);
}