Examples with StructuredRow io.cdap.cdap.spi.data.StructuredRow used on opensource projects

Example 66 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class AppMetadataStore method scanApplications.

/**
 * Scans applications. Allows to optionally set namespace / filters and implement pagination. For pagination
 * set {@link ScanApplicationsRequest#getScanFrom()} to the last application id of the previous page.
 *
 * @param request parameters defining filters and sorting
 * @param func a {@link Function} to consume application metadata entries generated by the scan. The boolean
 *             value returned is {@code true}, the scan will continue; otherwise the scan will stop and return.
 *             Note that the parameter is a {@link Map.Entry} to allow lazy deserialization of
 *             {@link ApplicationMeta} and it should not be replaced with {@link BiFunction}.
 * @see ScanApplicationsRequest#builder(ScanApplicationsRequest) to create a next page / batch request
 * @throws IOException if failed to scan the storage
 */
public void scanApplications(ScanApplicationsRequest request, Function<Map.Entry<ApplicationId, ApplicationMeta>, Boolean> func) throws IOException {
    Range.Bound startBound = Range.Bound.INCLUSIVE;
    Collection<Field<?>> startFields = request.getNamespaceId() == null ? Collections.emptyList() : Collections.singletonList(Fields.stringField(StoreDefinition.AppMetadataStore.NAMESPACE_FIELD, request.getNamespaceId().getNamespace()));
    Range.Bound endBound = Range.Bound.INCLUSIVE;
    Collection<Field<?>> endFields = startFields;
    if (request.getScanFrom() != null) {
        if (request.getNamespaceId() != null && !request.getNamespaceId().equals(request.getScanFrom().getNamespaceId())) {
            throw new IllegalArgumentException("Requested to start scan from application " + request.getScanFrom() + " that is outside of scan namespace " + request.getNamespaceId());
        }
        startBound = Range.Bound.EXCLUSIVE;
        startFields = getApplicationPrimaryKeys(request.getScanFrom());
    }
    if (request.getScanTo() != null) {
        if (request.getNamespaceId() != null && !request.getNamespaceId().equals(request.getScanTo().getNamespaceId())) {
            throw new IllegalArgumentException("Requested to finish scan at application " + request.getScanTo() + " that is outside of scan namespace " + request.getNamespaceId());
        }
        endBound = Range.Bound.EXCLUSIVE;
        endFields = getApplicationPrimaryKeys(request.getScanTo());
    }
    Range range;
    if (request.getSortOrder() == SortOrder.ASC) {
        range = Range.create(startFields, startBound, endFields, endBound);
    } else {
        range = Range.create(endFields, endBound, startFields, startBound);
    }
    // As of now this is where we push filter to. it does not go to the StructuredTable,
    // but we don't deserialize ApplicationMeta unless needed
    Predicate<AppScanEntry> scanEntryPredicate = e -> true;
    for (ApplicationFilter filter : request.getFilters()) {
        if (filter instanceof ApplicationFilter.ApplicationIdFilter) {
            scanEntryPredicate = scanEntryPredicate.and(e -> ((ApplicationFilter.ApplicationIdFilter) filter).test(e.getKey()));
        } else if (filter instanceof ApplicationFilter.ArtifactIdFilter) {
            scanEntryPredicate = scanEntryPredicate.and(e -> ((ApplicationFilter.ArtifactIdFilter) filter).test(e.getValue().getSpec().getArtifactId()));
        } else {
            throw new UnsupportedOperationException("Application filter " + filter + " is not supported");
        }
    }
    StructuredTable table = getApplicationSpecificationTable();
    int limit = request.getLimit();
    try (CloseableIterator<StructuredRow> iterator = table.scan(range, Integer.MAX_VALUE, request.getSortOrder())) {
        boolean keepScanning = true;
        while (iterator.hasNext() && keepScanning && limit > 0) {
            StructuredRow row = iterator.next();
            AppScanEntry scanEntry = new AppScanEntry(row);
            if (scanEntryPredicate.test(scanEntry)) {
                keepScanning = func.apply(scanEntry);
                limit--;
            }
        }
    }
}

Example 67 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class WorkflowTable method getRecord.

@Nullable
WorkflowRunRecord getRecord(WorkflowId id, String pid) throws IOException {
    RunId runId = RunIds.fromString(pid);
    long startTime = RunIds.getTime(runId, TimeUnit.SECONDS);
    List<Field<?>> fields = getPrimaryKeyFields(id, startTime);
    Optional<StructuredRow> indexRow = table.read(fields);
    if (!indexRow.isPresent()) {
        return null;
    }
    return getRunRecordFromRow(indexRow.get());
}

Example 68 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class WorkflowTable method getNeighbors.

/**
 * Returns a map of WorkflowRunId to WorkflowRunRecord that are close to the WorkflowRunId provided by the user.
 *
 * @param id The workflow
 * @param runId The runid of the workflow
 * @param limit The limit on each side of the run that we want to see into
 * @param timeInterval The time interval that we want the results to be spaced apart
 * @return A Map of WorkflowRunId to the corresponding Workflow Run Record. A map is used so that duplicates of
 * the WorkflowRunRecord are not obtained
 */
private Map<String, WorkflowRunRecord> getNeighbors(WorkflowId id, RunId runId, int limit, long timeInterval) throws IOException {
    long startTime = RunIds.getTime(runId, TimeUnit.SECONDS);
    Map<String, WorkflowRunRecord> workflowRunRecords = new HashMap<>();
    int i = -limit;
    long prevStartTime = startTime - (limit * timeInterval);
    // The loop iterates across the range that is startTime - (limit * timeInterval) to
    // startTime + (limit * timeInterval) since we want to capture all runs that started in this range.
    // Since we want to stop getting the same key, we have the prevStartTime become 1 more than the time at which
    // the last record was found if the (interval * the count of the loop) is less than the time.
    long upperBound = startTime + (limit * timeInterval);
    while (prevStartTime <= upperBound) {
        List<Field<?>> lowerBoundFields = getPrimaryKeyFields(id, prevStartTime);
        List<Field<?>> upperBoundFields = getPrimaryKeyFields(id, upperBound);
        // last primary key which is numeric.
        try (CloseableIterator<StructuredRow> iterator = table.scan(Range.create(lowerBoundFields, Range.Bound.INCLUSIVE, upperBoundFields, Range.Bound.INCLUSIVE), 1)) {
            if (!iterator.hasNext()) {
                return workflowRunRecords;
            }
            StructuredRow indexRow = iterator.next();
            long timeOfNextRecord = indexRow.getLong(StoreDefinition.WorkflowStore.START_TIME_FIELD);
            workflowRunRecords.put(indexRow.getString(StoreDefinition.WorkflowStore.RUN_ID_FIELD), getRunRecordFromRow(indexRow));
            prevStartTime = startTime + (i * timeInterval) < timeOfNextRecord ? timeOfNextRecord + 1 : startTime + (i * timeInterval);
            i++;
        }
    }
    return workflowRunRecords;
}

Example 69 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class LineageTable method getAccessTimesForRun.

/**
 * @return a set of access times (for program and data it accesses) associated with a program run.
 */
@VisibleForTesting
public List<Long> getAccessTimesForRun(ProgramRunId run) throws IOException {
    ImmutableList.Builder<Long> builder = ImmutableList.builder();
    List<Field<?>> prefix = getRunScanStartKey(run);
    try (CloseableIterator<StructuredRow> iterator = getProgramTable().scan(Range.singleton(prefix), Integer.MAX_VALUE)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            if (run.getRun().equals(row.getString(StoreDefinition.LineageStore.RUN_FIELD))) {
                builder.add(row.getLong(StoreDefinition.LineageStore.ACCESS_TIME_FIELD));
            }
        }
    }
    return builder.build();
}

Example 70 with StructuredRow

use of io.cdap.cdap.spi.data.StructuredRow in project cdap by cdapio.

the class FieldLineageTable method getSourceFields.

private Set<String> getSourceFields(EndPoint endPoint, long start, long end) throws IOException {
    Set<Long> checksums = getChecksumsWithProgramRunsInRange(OUTGOING_DIRECTION_MARKER, endPoint, start, end).keySet();
    Set<String> fields = new HashSet<>();
    for (long checksum : checksums) {
        List<Field<?>> prefix = getSummaryPrefix(checksum, OUTGOING_DIRECTION_MARKER, endPoint);
        try (CloseableIterator<StructuredRow> iterator = getSummaryFieldsTable().scan(Range.singleton(prefix), Integer.MAX_VALUE)) {
            while (iterator.hasNext()) {
                StructuredRow row = iterator.next();
                fields.add(row.getString(StoreDefinition.FieldLineageStore.ENDPOINT_FIELD));
            }
        }
    }
    return fields;
}