Examples with QueryByPages com.vmware.photon.controller.model.query.QueryUtils.QueryByPages used on opensource projects

Example 16 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class TestAzureEnumerationTask method getSubnetStates.

/**
 * Get all SubnetStates within passed NetworkState. In other words, get all subnet states that
 * refer the network state passed.
 */
// TODO : Duplicated from AWS TestUtils. Please advice where to put common test utils
public static List<SubnetState> getSubnetStates(VerificationHost host, NetworkState networkState) {
    Query queryForReferrers = QueryUtils.queryForReferrers(networkState.documentSelfLink, SubnetState.class, SubnetState.FIELD_NAME_NETWORK_LINK);
    QueryByPages<SubnetState> querySubnetStatesReferrers = new QueryByPages<>(host, queryForReferrers, SubnetState.class, networkState.tenantLinks, networkState.endpointLink);
    DeferredResult<List<SubnetState>> subnetDR = querySubnetStatesReferrers.collectDocuments(Collectors.toList());
    return waitToComplete(subnetDR);
}

Example 17 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class EndpointEnumerationProcess method disassociateLocalResourceStates.

/**
 * Disassociate stale local resource states. The logic works by recording a timestamp when
 * enumeration starts. This timestamp is used to lookup resources which have not been touched as
 * part of current enumeration cycle. Resources not associated with any endpointLink will be
 * removed by the groomer task.
 * <p>
 * Here is the list of criteria used to locate the stale local resources states:
 * <ul>
 * <li>Add local documents' kind:
 * {@code qBuilder.addKindFieldClause(context.localStateClass)}</li>
 * <li>Add time stamp older than current enumeration cycle:
 * {@code qBuilder.addRangeClause(ServiceDocument.FIELD_NAME_UPDATE_TIME_MICROS, createLessThanRange(context.enumStartTimeInMicros))}</li>
 * <li>Add {@code tenantLinks} and {@code endpointLink} criteria as defined by
 * {@code QueryTemplate}</li>
 * <li>Add descendant specific criteria as defined by
 * {@link #customizeLocalStatesQuery(com.vmware.xenon.services.common.QueryTask.Query.Builder)}</li>
 * </ul>
 */
protected DeferredResult<T> disassociateLocalResourceStates(T context) {
    final String msg = "Disassociate %ss that no longer exist in the endpoint: %s";
    context.service.logFine(() -> String.format(msg, context.localStateClass.getSimpleName(), "STARTING"));
    Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(context.localStateClass).addRangeClause(ServiceDocument.FIELD_NAME_UPDATE_TIME_MICROS, createLessThanRange(context.enumStartTimeInMicros));
    if (getEndpointRegion() != null) {
        // Limit documents within end-point region
        qBuilder.addFieldClause(ResourceState.FIELD_NAME_REGION_ID, getEndpointRegion());
    }
    if (!this.enumExternalResourcesIds.isEmpty() && this.enumExternalResourcesIds.size() <= MAX_RESOURCES_TO_QUERY_ON_DELETE) {
        // do not load resources from enumExternalResourcesIds
        qBuilder.addInClause(ResourceState.FIELD_NAME_ID, this.enumExternalResourcesIds, Occurance.MUST_NOT_OCCUR);
    }
    // Delegate to descendants to any doc specific criteria
    customizeLocalStatesQuery(qBuilder);
    QueryByPages<LOCAL_STATE> queryLocalStates = new QueryByPages<>(context.service.getHost(), qBuilder.build(), context.localStateClass, isApplyInfraFields() ? context.endpointState.tenantLinks : null, isApplyEndpointLink() ? context.endpointState.documentSelfLink : null, null).setQueryTaskTenantLinks(context.endpointState.tenantLinks);
    queryLocalStates.setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
    List<DeferredResult<Operation>> disassociateDRs = new ArrayList<>();
    // Delete stale resources.
    return queryLocalStates.queryDocuments(localState -> {
        if (!shouldDelete(localState)) {
            return;
        }
        // Deleting the localResourceState is done by disassociating the
        // endpointLink from the localResourceState. If the localResourceState
        // isn't associated with any other endpointLink, it should be eventually
        // deleted by the groomer task
        Operation disassociateOp = PhotonModelUtils.createRemoveEndpointLinksOperation(context.service, context.endpointState.documentSelfLink, localState);
        if (disassociateOp == null) {
            return;
        }
        // NOTE: The original Op is set with completion that must be executed.
        // Since sendWithDeferredResult is used we must manually call it, otherwise it's
        // just ignored.
        CompletionHandler disassociateOpCompletion = disassociateOp.getCompletion();
        DeferredResult<Operation> disassociateDR = context.service.sendWithDeferredResult(disassociateOp).whenComplete(disassociateOpCompletion::handle).whenComplete((o, e) -> {
            final String message = "Disassociate stale %s state";
            if (e != null) {
                context.service.logWarning(message + ": FAILED with %s", localState.documentSelfLink, Utils.toString(e));
            } else {
                context.service.log(Level.FINEST, message + ": SUCCESS", localState.documentSelfLink);
            }
        });
        disassociateDRs.add(disassociateDR);
    }).thenCompose(ignore -> DeferredResult.allOf(disassociateDRs)).thenApply(ignore -> context);
}

Example 18 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class TestAWSImageEnumerationTask method afterTest.

@After
public final void afterTest() throws Throwable {
    QueryByPages<ImageState> queryAll = new QueryByPages<ImageState>(getHost(), Builder.create().addKindFieldClause(ImageState.class).build(), ImageState.class, null);
    queryAll.setMaxPageSize(QueryUtils.DEFAULT_MAX_RESULT_LIMIT);
    AtomicInteger counter = new AtomicInteger(0);
    waitToComplete(queryAll.queryLinks(imageLink -> {
        try {
            deleteServiceSynchronously(imageLink);
            counter.incrementAndGet();
        } catch (Throwable e) {
            throw new RuntimeException(e);
        }
    }));
    getHost().log(Level.INFO, "[" + this.currentTestName.getMethodName() + "] Deleted " + counter + " ImageStates");
}

Example 19 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method getLocalStorageAccountDescriptions.

/**
 * Query all storage descriptions for the cluster filtered by the received set of storage
 * account Ids
 */
private void getLocalStorageAccountDescriptions(StorageEnumContext context, StorageEnumStages next) {
    if (context.storageAccountsToUpdateCreate.isEmpty()) {
        context.subStage = StorageEnumStages.CREATE_STORAGE_DESCRIPTIONS;
        handleSubStage(context);
        return;
    }
    context.storageDescriptions.clear();
    Query.Builder qBuilder = Query.Builder.create().addKindFieldClause(StorageDescription.class);
    Query.Builder instanceIdFilterParentQuery = Query.Builder.create(Occurance.MUST_OCCUR);
    for (Map.Entry<String, StorageAccount> account : context.storageAccountsToUpdateCreate.entrySet()) {
        Query instanceIdFilter = Query.Builder.create(Occurance.SHOULD_OCCUR).addFieldClause(StorageDescription.FIELD_NAME_ID, canonizeId(account.getValue().id)).build();
        instanceIdFilterParentQuery.addClause(instanceIdFilter);
    }
    qBuilder.addClause(instanceIdFilterParentQuery.build());
    QueryByPages<StorageDescription> queryLocalStates = new QueryByPages<>(getHost(), qBuilder.build(), StorageDescription.class, context.parentCompute.tenantLinks, null, /* endpointLink */
    context.parentCompute.documentSelfLink).setMaxPageSize(QueryUtils.MAX_RESULT_LIMIT).setClusterType(ServiceTypeCluster.INVENTORY_SERVICE);
    queryLocalStates.collectDocuments(Collectors.toList()).whenComplete((sds, ex) -> {
        if (ex != null) {
            handleError(context, ex);
            return;
        }
        logFine(() -> String.format("Found %d matching storage descriptions for Azure" + " storage accounts", sds.size()));
        List<DeferredResult<AuthCredentialsServiceState>> results = sds.stream().map(sd -> {
            context.storageDescriptions.put(sd.id, sd);
            // populate connectionStrings
            if (!context.storageConnectionStrings.containsKey(sd.id)) {
                return loadStorageAuth(context, sd);
            } else {
                return DeferredResult.<AuthCredentialsServiceState>completed(null);
            }
        }).collect(Collectors.toList());
        DeferredResult.allOf(results).whenComplete((creds, e) -> {
            if (e != null) {
                logWarning(() -> String.format("Failed to get storage description" + " credentials: %s", e.getMessage()));
            }
            context.subStage = next;
            handleSubStage(context);
        });
    });
}

Example 20 with QueryByPages

use of com.vmware.photon.controller.model.query.QueryUtils.QueryByPages in project photon-model by vmware.

the class TestAzureImageEnumerationTask method testPublicImageEnumeration_all.

@Test
public void testPublicImageEnumeration_all() throws Throwable {
    Assume.assumeFalse(this.isMock);
    Assume.assumeTrue(this.enableLongRunning);
    // This test takes about 30 mins!
    getHost().setTimeoutSeconds((int) TimeUnit.MINUTES.toSeconds(40));
    ImageEnumerationTaskState task = kickOffImageEnumeration(this.endpointState, PUBLIC, null);
    // Validate at least 4.5K image states are created
    QueryByPages<ImageState> queryAll = new QueryByPages<ImageState>(getHost(), Builder.create().addKindFieldClause(ImageState.class).build(), ImageState.class, task.tenantLinks);
    queryAll.setMaxPageSize(QueryUtils.DEFAULT_MAX_RESULT_LIMIT);
    Long imagesCount = PhotonModelUtils.waitToComplete(queryAll.collectLinks(Collectors.counting()));
    Assert.assertTrue("Expected at least " + 4_500 + " images, but found only " + imagesCount, imagesCount > 4_500);
}