Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 1 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class RecordQueryPlanner method planFilterForInJoin.

private ScoredPlan planFilterForInJoin(@Nonnull PlanContext planContext, @Nonnull QueryComponent filter, boolean needOrdering) {
    planContext.rankComparisons = new RankComparisons(filter, planContext.indexes);
    List<ScoredPlan> intersectionCandidates = new ArrayList<>();
    ScoredPlan bestPlan = null;
    Index bestIndex = null;
    if (planContext.commonPrimaryKey != null) {
        bestPlan = planIndex(planContext, filter, null, planContext.commonPrimaryKey, intersectionCandidates);
    }
    for (Index index : planContext.indexes) {
        KeyExpression indexKeyExpression = indexKeyExpressionForPlan(planContext.commonPrimaryKey, index);
        ScoredPlan p = planIndex(planContext, filter, index, indexKeyExpression, intersectionCandidates);
        if (p != null) {
            // * need for type filtering if row scan with multiple types.
            if (isBetterThanOther(planContext, p, index, bestPlan, bestIndex)) {
                bestPlan = p;
                bestIndex = index;
            }
        }
    }
    if (bestPlan != null) {
        if (bestPlan.getNumNonSargables() > 0) {
            bestPlan = handleNonSargables(bestPlan, intersectionCandidates, planContext);
        }
        if (needOrdering) {
            bestPlan.planOrderingKey = PlanOrderingKey.forPlan(metaData, bestPlan.plan, planContext.commonPrimaryKey);
        }
    }
    return bestPlan;
}

Example 2 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class RecordQueryPlanner method planNoFilter.

@Nullable
private RecordQueryPlan planNoFilter(PlanContext planContext, KeyExpression sort, boolean sortReverse) {
    ScoredPlan bestPlan = null;
    Index bestIndex = null;
    if (sort == null) {
        bestPlan = planNoFilterNoSort(planContext, null);
    } else if (planContext.commonPrimaryKey != null) {
        bestPlan = planSortOnly(new CandidateScan(planContext, null, sortReverse), planContext.commonPrimaryKey, sort);
    }
    for (Index index : planContext.indexes) {
        ScoredPlan p;
        if (sort == null) {
            p = planNoFilterNoSort(planContext, index);
        } else {
            p = planSortOnly(new CandidateScan(planContext, index, sortReverse), indexKeyExpressionForPlan(planContext.commonPrimaryKey, index), sort);
        }
        if (p != null) {
            if (bestPlan == null || p.score > bestPlan.score || (p.score == bestPlan.score && compareIndexes(planContext, index, bestIndex) > 0)) {
                bestPlan = p;
                bestIndex = index;
            }
        }
    }
    if (bestPlan != null) {
        bestPlan = planRemoveDuplicates(planContext, bestPlan);
        if (bestPlan == null) {
            throw new RecordCoreException("A common primary key is required to remove duplicates");
        }
        return bestPlan.plan;
    }
    return null;
}

Example 3 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class RecordQueryPlanner method computeIndexFilters.

private ScoredPlan computeIndexFilters(@Nonnull PlanContext planContext, @Nonnull final ScoredPlan plan) {
    if (plan.plan instanceof RecordQueryPlanWithIndex) {
        final RecordQueryPlanWithIndex indexPlan = (RecordQueryPlanWithIndex) plan.plan;
        final Index index = metaData.getIndex(indexPlan.getIndexName());
        final Collection<RecordType> recordTypes = metaData.recordTypesForIndex(index);
        if (recordTypes.size() != 1) {
            return plan;
        }
        final RecordType recordType = Iterables.getOnlyElement(recordTypes);
        final List<QueryComponent> unsatisfiedFilters = new ArrayList<>(plan.unsatisfiedFilters);
        final AvailableFields availableFieldsFromIndex = AvailableFields.fromIndex(recordType, index, indexTypes, planContext.commonPrimaryKey);
        final List<QueryComponent> indexFilters = Lists.newArrayListWithCapacity(unsatisfiedFilters.size());
        final List<QueryComponent> residualFilters = Lists.newArrayListWithCapacity(unsatisfiedFilters.size());
        FilterVisitor.partitionFilters(unsatisfiedFilters, availableFieldsFromIndex, indexFilters, residualFilters, null);
        if (!indexFilters.isEmpty()) {
            return plan.withFilters(residualFilters, indexFilters);
        }
    }
    return plan;
}

Example 4 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class IndexingCommon method fillTargetIndexers.

private void fillTargetIndexers(@Nonnull List<Index> targetIndexes, @Nullable Collection<RecordType> recordTypes) {
    boolean presetTypes = false;
    if (recordTypes != null) {
        if (targetIndexes.size() > 1) {
            throw new IndexingBase.ValidationException("Can't use preset record types with multi target indexing");
        }
        presetTypes = true;
    }
    if (recordStoreBuilder.getMetaDataProvider() == null) {
        throw new MetaDataException("record store builder must include metadata");
    }
    final RecordMetaData metaData = recordStoreBuilder.getMetaDataProvider().getRecordMetaData();
    for (Index targetIndex : targetIndexes) {
        Collection<RecordType> types;
        if (presetTypes) {
            types = recordTypes;
        } else {
            types = metaData.recordTypesForIndex(targetIndex);
        }
        boolean isSynthetic = false;
        if (types.stream().anyMatch(RecordType::isSynthetic)) {
            types = new SyntheticRecordPlanner(metaData, new RecordStoreState(null, null)).storedRecordTypesForIndex(targetIndex, types);
            isSynthetic = true;
        }
        targetIndexContexts.add(new IndexContext(targetIndex, types, isSynthetic));
        allRecordTypes.addAll(types);
    }
}

Example 5 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method getRecordIfMissingIndex.

@Nullable
private CompletableFuture<FDBStoredRecord<Message>> getRecordIfMissingIndex(FDBRecordStore store, final RecordCursorResult<FDBStoredRecord<Message>> currResult) {
    final FDBStoredRecord<Message> rec = currResult.get();
    // return true if an index is missing and updated
    if (!common.getAllRecordTypes().contains(rec.getRecordType())) {
        return CompletableFuture.completedFuture(null);
    }
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    List<IndexEntry> indexEntryNoPKs = maintainer.filteredIndexEntries(rec);
    if (indexEntryNoPKs == null) {
        return CompletableFuture.completedFuture(null);
    }
    return AsyncUtil.getAll(indexEntryNoPKs.stream().map(entry -> {
        // should I convert it to a single nested statement?
        final IndexEntry indexEntry = new IndexEntry(index, FDBRecordStoreBase.indexEntryKey(index, entry.getKey(), rec.getPrimaryKey()), entry.getValue());
        final Tuple valueKey = indexEntry.getKey();
        final byte[] keyBytes = maintainer.getIndexSubspace().pack(valueKey);
        return maintainer.state.transaction.get(keyBytes).thenApply(indexVal -> indexVal == null ? valueKey : null);
    }).collect(Collectors.toList())).thenApply(list -> {
        List<Tuple> missingIndexesKeys = list.stream().filter(Objects::nonNull).collect(Collectors.toList());
        if (missingIndexesKeys.isEmpty()) {
            return null;
        }
        // (Maybe) report an error and (maybe) return this record to be index
        if (LOGGER.isWarnEnabled() && logWarningCounter > 0) {
            logWarningCounter--;
            LOGGER.warn(KeyValueLogMessage.build("Scrubber: missing index entry", LogMessageKeys.KEY, rec.getPrimaryKey().toString(), LogMessageKeys.INDEX_KEY, missingIndexesKeys.toString()).addKeysAndValues(common.indexLogMessageKeyValues()).toString());
        }
        missingCount.incrementAndGet();
        final FDBStoreTimer timer = getRunner().getTimer();
        timerIncrement(timer, FDBStoreTimer.Counts.INDEX_SCRUBBER_MISSING_ENTRIES);
        if (scrubbingPolicy.allowRepair()) {
            // record to be indexed
            return rec;
        }
        // report only mode
        return null;
    });
}