Search in sources :

Example 1 with ApplicationFilter

use of io.cdap.cdap.app.store.ApplicationFilter in project cdap by caskdata.

the class AppMetadataStore method scanApplications.

/**
 * Scans applications. Allows to optionally set namespace / filters and implement pagination. For pagination
 * set {@link ScanApplicationsRequest#getScanFrom()} to the last application id of the previous page.
 *
 * @param request parameters defining filters and sorting
 * @param func a {@link Function} to consume application metadata entries generated by the scan. The boolean
 *             value returned is {@code true}, the scan will continue; otherwise the scan will stop and return.
 *             Note that the parameter is a {@link Map.Entry} to allow lazy deserialization of
 *             {@link ApplicationMeta} and it should not be replaced with {@link BiFunction}.
 * @see ScanApplicationsRequest#builder(ScanApplicationsRequest) to create a next page / batch request
 * @throws IOException if failed to scan the storage
 */
public void scanApplications(ScanApplicationsRequest request, Function<Map.Entry<ApplicationId, ApplicationMeta>, Boolean> func) throws IOException {
    Range.Bound startBound = Range.Bound.INCLUSIVE;
    Collection<Field<?>> startFields = request.getNamespaceId() == null ? Collections.emptyList() : Collections.singletonList(Fields.stringField(StoreDefinition.AppMetadataStore.NAMESPACE_FIELD, request.getNamespaceId().getNamespace()));
    Range.Bound endBound = Range.Bound.INCLUSIVE;
    Collection<Field<?>> endFields = startFields;
    if (request.getScanFrom() != null) {
        if (request.getNamespaceId() != null && !request.getNamespaceId().equals(request.getScanFrom().getNamespaceId())) {
            throw new IllegalArgumentException("Requested to start scan from application " + request.getScanFrom() + " that is outside of scan namespace " + request.getNamespaceId());
        }
        startBound = Range.Bound.EXCLUSIVE;
        startFields = getApplicationPrimaryKeys(request.getScanFrom());
    }
    if (request.getScanTo() != null) {
        if (request.getNamespaceId() != null && !request.getNamespaceId().equals(request.getScanTo().getNamespaceId())) {
            throw new IllegalArgumentException("Requested to finish scan at application " + request.getScanTo() + " that is outside of scan namespace " + request.getNamespaceId());
        }
        endBound = Range.Bound.EXCLUSIVE;
        endFields = getApplicationPrimaryKeys(request.getScanTo());
    }
    Range range;
    if (request.getSortOrder() == SortOrder.ASC) {
        range = Range.create(startFields, startBound, endFields, endBound);
    } else {
        range = Range.create(endFields, endBound, startFields, startBound);
    }
    // As of now this is where we push filter to. it does not go to the StructuredTable,
    // but we don't deserialize ApplicationMeta unless needed
    Predicate<AppScanEntry> scanEntryPredicate = e -> true;
    for (ApplicationFilter filter : request.getFilters()) {
        if (filter instanceof ApplicationFilter.ApplicationIdFilter) {
            scanEntryPredicate = scanEntryPredicate.and(e -> ((ApplicationFilter.ApplicationIdFilter) filter).test(e.getKey()));
        } else if (filter instanceof ApplicationFilter.ArtifactIdFilter) {
            scanEntryPredicate = scanEntryPredicate.and(e -> ((ApplicationFilter.ArtifactIdFilter) filter).test(e.getValue().getSpec().getArtifactId()));
        } else {
            throw new UnsupportedOperationException("Application filter " + filter + " is not supported");
        }
    }
    StructuredTable table = getApplicationSpecificationTable();
    int limit = request.getLimit();
    try (CloseableIterator<StructuredRow> iterator = table.scan(range, Integer.MAX_VALUE, request.getSortOrder())) {
        boolean keepScanning = true;
        while (iterator.hasNext() && keepScanning && limit > 0) {
            StructuredRow row = iterator.next();
            AppScanEntry scanEntry = new AppScanEntry(row);
            if (scanEntryPredicate.test(scanEntry)) {
                keepScanning = func.apply(scanEntry);
                limit--;
            }
        }
    }
}
Also used : Arrays(java.util.Arrays) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) BiFunction(java.util.function.BiFunction) LoggerFactory(org.slf4j.LoggerFactory) Bytes(io.cdap.cdap.api.common.Bytes) Fields(io.cdap.cdap.spi.data.table.field.Fields) GsonBuilder(com.google.gson.GsonBuilder) ProgramRunCluster(io.cdap.cdap.proto.ProgramRunCluster) ScanApplicationsRequest(io.cdap.cdap.app.store.ScanApplicationsRequest) DatasetId(io.cdap.cdap.proto.id.DatasetId) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Gson(com.google.gson.Gson) Map(java.util.Map) Field(io.cdap.cdap.spi.data.table.field.Field) RunId(org.apache.twill.api.RunId) BasicWorkflowToken(io.cdap.cdap.internal.app.runtime.workflow.BasicWorkflowToken) SortOrder(io.cdap.cdap.spi.data.SortOrder) BasicThrowable(io.cdap.cdap.proto.BasicThrowable) StoreDefinition(io.cdap.cdap.store.StoreDefinition) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) Set(java.util.Set) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) StructuredTableContext(io.cdap.cdap.spi.data.StructuredTableContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) ApplicationSpecificationAdapter(io.cdap.cdap.internal.app.ApplicationSpecificationAdapter) Objects(java.util.Objects) AbstractCloseableIterator(io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator) List(java.util.List) Type(java.lang.reflect.Type) Optional(java.util.Optional) Constants(io.cdap.cdap.common.conf.Constants) ProfileId(io.cdap.cdap.proto.id.ProfileId) ProgramOptionConstants(io.cdap.cdap.internal.app.runtime.ProgramOptionConstants) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) HashMap(java.util.HashMap) TypeToken(com.google.common.reflect.TypeToken) ProgramType(io.cdap.cdap.proto.ProgramType) Function(java.util.function.Function) JsonReader(com.google.gson.stream.JsonReader) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) WorkflowToken(io.cdap.cdap.api.workflow.WorkflowToken) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ImmutableList(com.google.common.collect.ImmutableList) BiConsumer(java.util.function.BiConsumer) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) LinkedHashSet(java.util.LinkedHashSet) Nullable(javax.annotation.Nullable) WorkflowNodeStateDetail(io.cdap.cdap.proto.WorkflowNodeStateDetail) Logger(org.slf4j.Logger) RunIds(io.cdap.cdap.common.app.RunIds) ApplicationFilter(io.cdap.cdap.app.store.ApplicationFilter) ProgramId(io.cdap.cdap.proto.id.ProgramId) IOException(java.io.IOException) BadRequestException(io.cdap.cdap.common.BadRequestException) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) ProgramRunClusterStatus(io.cdap.cdap.proto.ProgramRunClusterStatus) TableNotFoundException(io.cdap.cdap.spi.data.TableNotFoundException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) JsonToken(com.google.gson.stream.JsonToken) StringReader(java.io.StringReader) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) Range(io.cdap.cdap.spi.data.table.field.Range) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) Range(io.cdap.cdap.spi.data.table.field.Range) Field(io.cdap.cdap.spi.data.table.field.Field) ApplicationFilter(io.cdap.cdap.app.store.ApplicationFilter)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 Sets (com.google.common.collect.Sets)1 TypeToken (com.google.common.reflect.TypeToken)1 Gson (com.google.gson.Gson)1 GsonBuilder (com.google.gson.GsonBuilder)1 JsonReader (com.google.gson.stream.JsonReader)1 JsonToken (com.google.gson.stream.JsonToken)1 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)1 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)1 Bytes (io.cdap.cdap.api.common.Bytes)1 AbstractCloseableIterator (io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator)1 CloseableIterator (io.cdap.cdap.api.dataset.lib.CloseableIterator)1 WorkflowToken (io.cdap.cdap.api.workflow.WorkflowToken)1 ApplicationFilter (io.cdap.cdap.app.store.ApplicationFilter)1 ScanApplicationsRequest (io.cdap.cdap.app.store.ScanApplicationsRequest)1 BadRequestException (io.cdap.cdap.common.BadRequestException)1 RunIds (io.cdap.cdap.common.app.RunIds)1