Examples with ScanApplicationsRequest io.cdap.cdap.app.store.ScanApplicationsRequest used on opensource projects

Example 1 with ScanApplicationsRequest

use of io.cdap.cdap.app.store.ScanApplicationsRequest in project cdap by caskdata.

the class DefaultStoreTest method testScanApplicationsWithNamespace.

public void testScanApplicationsWithNamespace(Store store) {
    ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
    int count = 100;
    for (int i = 0; i < count / 2; i++) {
        String appName = "test" + (2 * i);
        store.addApplication(new ApplicationId(NamespaceId.DEFAULT.getNamespace(), appName), appSpec);
        appName = "test" + (2 * i + 1);
        store.addApplication(new ApplicationId(NamespaceId.CDAP.getNamespace(), appName), appSpec);
    }
    List<ApplicationId> apps = new ArrayList<ApplicationId>();
    ScanApplicationsRequest request = ScanApplicationsRequest.builder().setNamespaceId(NamespaceId.CDAP).build();
    store.scanApplications(request, 20, (appId, spec) -> {
        apps.add(appId);
    });
    Assert.assertEquals(count / 2, apps.size());
    // Reverse
    List<ApplicationId> reverseApps = new ArrayList<>();
    request = ScanApplicationsRequest.builder().setNamespaceId(NamespaceId.CDAP).setSortOrder(SortOrder.DESC).build();
    store.scanApplications(request, 20, (appId, spec) -> reverseApps.add(appId));
    Assert.assertEquals(Lists.reverse(apps), reverseApps);
    // Second page
    int firstPageSize = 10;
    List<ApplicationId> restartApps = new ArrayList<>();
    request = ScanApplicationsRequest.builder().setNamespaceId(NamespaceId.CDAP).setScanFrom(apps.get(firstPageSize - 1)).build();
    store.scanApplications(request, 20, (appId, spec) -> restartApps.add(appId));
    Assert.assertEquals(apps.subList(firstPageSize, apps.size()), restartApps);
}

Example 2 with ScanApplicationsRequest

use of io.cdap.cdap.app.store.ScanApplicationsRequest in project cdap by caskdata.

the class DefaultStore method scanApplications.

@Override
public boolean scanApplications(ScanApplicationsRequest request, int txBatchSize, BiConsumer<ApplicationId, ApplicationSpecification> consumer) {
    AtomicReference<ScanApplicationsRequest> requestRef = new AtomicReference<>(request);
    AtomicReference<ApplicationId> lastKey = new AtomicReference<>();
    AtomicInteger currentLimit = new AtomicInteger(request.getLimit());
    while (currentLimit.get() > 0) {
        AtomicInteger count = new AtomicInteger();
        try {
            TransactionRunners.run(transactionRunner, context -> {
                getAppMetadataStore(context).scanApplications(requestRef.get(), entry -> {
                    lastKey.set(entry.getKey());
                    currentLimit.decrementAndGet();
                    consumer.accept(entry.getKey(), entry.getValue().getSpec());
                    return count.incrementAndGet() < txBatchSize && currentLimit.get() > 0;
                });
            });
        } catch (UnsupportedOperationException e) {
            if (requestRef.get().getSortOrder() != SortOrder.DESC || count.get() != 0) {
                throw e;
            }
            scanApplicationwWithReorder(requestRef.get(), txBatchSize, consumer);
        }
        if (lastKey.get() == null) {
            break;
        }
        ScanApplicationsRequest nextBatchRequest = ScanApplicationsRequest.builder(requestRef.get()).setScanFrom(lastKey.get()).setLimit(currentLimit.get()).build();
        requestRef.set(nextBatchRequest);
        lastKey.set(null);
    }
    return currentLimit.get() == 0;
}

Example 3 with ScanApplicationsRequest

use of io.cdap.cdap.app.store.ScanApplicationsRequest in project cdap by caskdata.

the class AppLifecycleHttpHandler method getAllApps.

/**
 * Returns a list of applications associated with a namespace.
 */
@GET
@Path("/apps")
public void getAllApps(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @QueryParam("artifactName") String artifactName, @QueryParam("artifactVersion") String artifactVersion, @QueryParam("pageToken") String pageToken, @QueryParam("pageSize") Integer pageSize, @QueryParam("orderBy") SortOrder orderBy, @QueryParam("nameFilter") String nameFilter) throws Exception {
    NamespaceId namespace = validateNamespace(namespaceId);
    Set<String> names = new HashSet<>();
    if (!Strings.isNullOrEmpty(artifactName)) {
        for (String name : Splitter.on(',').split(artifactName)) {
            names.add(name);
        }
    }
    if (Optional.ofNullable(pageSize).orElse(0) != 0) {
        JsonPaginatedListResponder.respond(GSON, responder, APP_LIST_PAGINATED_KEY, jsonListResponder -> {
            AtomicReference<ApplicationRecord> lastRecord = new AtomicReference<>(null);
            ScanApplicationsRequest scanRequest = getScanRequest(namespaceId, artifactVersion, pageToken, pageSize, orderBy, nameFilter, names);
            boolean pageLimitReached = applicationLifecycleService.scanApplications(scanRequest, appDetail -> {
                ApplicationRecord record = new ApplicationRecord(appDetail);
                jsonListResponder.send(record);
                lastRecord.set(record);
            });
            ApplicationRecord record = lastRecord.get();
            return !pageLimitReached || record == null ? null : record.getName() + EntityId.IDSTRING_PART_SEPARATOR + record.getAppVersion();
        });
    } else {
        ScanApplicationsRequest scanRequest = getScanRequest(namespaceId, artifactVersion, pageToken, null, orderBy, nameFilter, names);
        JsonWholeListResponder.respond(GSON, responder, jsonListResponder -> applicationLifecycleService.scanApplications(scanRequest, d -> jsonListResponder.send(new ApplicationRecord(d))));
    }
}

Example 4 with ScanApplicationsRequest

use of io.cdap.cdap.app.store.ScanApplicationsRequest in project cdap by caskdata.

the class CDAPEntities method collect.

@Override
public void collect() throws Exception {
    reset();
    List<NamespaceMeta> namespaceMetas = nsQueryAdmin.list();
    namespaces = namespaceMetas.size();
    artifacts += artifactRepository.getArtifactSummaries(NamespaceId.SYSTEM, false).size();
    for (NamespaceMeta meta : namespaceMetas) {
        ScanApplicationsRequest scanApplicationsRequest = ScanApplicationsRequest.builder().setNamespaceId(meta.getNamespaceId()).build();
        appLifecycleService.scanApplications(scanApplicationsRequest, d -> {
            this.apps++;
            programs += d.getPrograms().size();
        });
        artifacts += artifactRepository.getArtifactSummaries(meta.getNamespaceId(), false).size();
        datasets += dsFramework.getInstances(meta.getNamespaceId()).size();
    }
}

Example 5 with ScanApplicationsRequest

use of io.cdap.cdap.app.store.ScanApplicationsRequest in project cdap by caskdata.

the class AppMetadataStore method scanApplications.

/**
 * Scans applications. Allows to optionally set namespace / filters and implement pagination. For pagination
 * set {@link ScanApplicationsRequest#getScanFrom()} to the last application id of the previous page.
 *
 * @param request parameters defining filters and sorting
 * @param func a {@link Function} to consume application metadata entries generated by the scan. The boolean
 *             value returned is {@code true}, the scan will continue; otherwise the scan will stop and return.
 *             Note that the parameter is a {@link Map.Entry} to allow lazy deserialization of
 *             {@link ApplicationMeta} and it should not be replaced with {@link BiFunction}.
 * @see ScanApplicationsRequest#builder(ScanApplicationsRequest) to create a next page / batch request
 * @throws IOException if failed to scan the storage
 */
public void scanApplications(ScanApplicationsRequest request, Function<Map.Entry<ApplicationId, ApplicationMeta>, Boolean> func) throws IOException {
    Range.Bound startBound = Range.Bound.INCLUSIVE;
    Collection<Field<?>> startFields = request.getNamespaceId() == null ? Collections.emptyList() : Collections.singletonList(Fields.stringField(StoreDefinition.AppMetadataStore.NAMESPACE_FIELD, request.getNamespaceId().getNamespace()));
    Range.Bound endBound = Range.Bound.INCLUSIVE;
    Collection<Field<?>> endFields = startFields;
    if (request.getScanFrom() != null) {
        if (request.getNamespaceId() != null && !request.getNamespaceId().equals(request.getScanFrom().getNamespaceId())) {
            throw new IllegalArgumentException("Requested to start scan from application " + request.getScanFrom() + " that is outside of scan namespace " + request.getNamespaceId());
        }
        startBound = Range.Bound.EXCLUSIVE;
        startFields = getApplicationPrimaryKeys(request.getScanFrom());
    }
    if (request.getScanTo() != null) {
        if (request.getNamespaceId() != null && !request.getNamespaceId().equals(request.getScanTo().getNamespaceId())) {
            throw new IllegalArgumentException("Requested to finish scan at application " + request.getScanTo() + " that is outside of scan namespace " + request.getNamespaceId());
        }
        endBound = Range.Bound.EXCLUSIVE;
        endFields = getApplicationPrimaryKeys(request.getScanTo());
    }
    Range range;
    if (request.getSortOrder() == SortOrder.ASC) {
        range = Range.create(startFields, startBound, endFields, endBound);
    } else {
        range = Range.create(endFields, endBound, startFields, startBound);
    }
    // As of now this is where we push filter to. it does not go to the StructuredTable,
    // but we don't deserialize ApplicationMeta unless needed
    Predicate<AppScanEntry> scanEntryPredicate = e -> true;
    for (ApplicationFilter filter : request.getFilters()) {
        if (filter instanceof ApplicationFilter.ApplicationIdFilter) {
            scanEntryPredicate = scanEntryPredicate.and(e -> ((ApplicationFilter.ApplicationIdFilter) filter).test(e.getKey()));
        } else if (filter instanceof ApplicationFilter.ArtifactIdFilter) {
            scanEntryPredicate = scanEntryPredicate.and(e -> ((ApplicationFilter.ArtifactIdFilter) filter).test(e.getValue().getSpec().getArtifactId()));
        } else {
            throw new UnsupportedOperationException("Application filter " + filter + " is not supported");
        }
    }
    StructuredTable table = getApplicationSpecificationTable();
    int limit = request.getLimit();
    try (CloseableIterator<StructuredRow> iterator = table.scan(range, Integer.MAX_VALUE, request.getSortOrder())) {
        boolean keepScanning = true;
        while (iterator.hasNext() && keepScanning && limit > 0) {
            StructuredRow row = iterator.next();
            AppScanEntry scanEntry = new AppScanEntry(row);
            if (scanEntryPredicate.test(scanEntry)) {
                keepScanning = func.apply(scanEntry);
                limit--;
            }
        }
    }
}