Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 11 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlanner method storedRecordTypesForIndex.

/**
 * Determine what stored record types would be need to scanned in order to rebuild a given index.
 *
 * From those scans, queries will be executed to load other record types to complete the synthesis.
 * <p>
 * In cases such as full outer join, there is no single record type from which all joins can be produced.
 * @param index the index that needs to be built
 * @param recordTypes a subset of the index's record types or {@code null} for all
 * @return a set of stored record types that are sufficient to generate the synthesized records for the index
 */
public Set<RecordType> storedRecordTypesForIndex(@Nonnull Index index, @Nullable Collection<RecordType> recordTypes) {
    if (recordTypes == null) {
        recordTypes = recordMetaData.recordTypesForIndex(index);
    }
    Set<RecordType> result = new HashSet<>();
    for (RecordType recordType : recordTypes) {
        if (!(recordType instanceof JoinedRecordType)) {
            throw unknownSyntheticType(recordType);
        }
        JoinedRecordType joinedRecordType = (JoinedRecordType) recordType;
        Optional<JoinedRecordType.JoinConstituent> maybeConstituent = joinedRecordType.getConstituents().stream().filter(c -> !c.isOuterJoined()).findFirst();
        if (maybeConstituent.isPresent()) {
            result.add(maybeConstituent.get().getRecordType());
        } else {
            for (JoinedRecordType.JoinConstituent joinConstituent : joinedRecordType.getConstituents()) {
                result.add(joinConstituent.getRecordType());
            }
        }
    }
    return result;
}

Example 12 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method performIndexOperationAsync.

public CompletableFuture<IndexOperationResult> performIndexOperationAsync(@Nonnull String indexName, @Nonnull IndexOperation operation) {
    final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
    final Index index = metaData.getIndex(indexName);
    return getIndexMaintainer(index).performOperation(operation);
}

Example 13 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method getPrimaryKeyBoundaries.

/**
 * Return a cursor of boundaries separating the key ranges maintained by each FDB server. This information can be
 * useful for splitting a large task (e.g., rebuilding an index for a large record store) into smaller tasks (e.g.,
 * rebuilding the index for records in certain primary key ranges) more evenly so that they can be executed in a
 * parallel fashion efficiently. The returned boundaries are an estimate from FDB's locality API and may not
 * represent the exact boundary locations at any database version.
 * <p>
 * The boundaries are returned as a cursor which is sorted and does not contain any duplicates. The first element of
 * the list is greater than or equal to <code>low</code>, and the last element is less than or equal to
 * <code>high</code>.
 * <p>
 * This implementation may not work when there are too many shard boundaries to complete in a single transaction.
 * <p>
 * Note: the returned cursor is blocking and must not be used in an asynchronous context
 *
 * @param low low endpoint of primary key range (inclusive)
 * @param high high endpoint of primary key range (exclusive)
 * @return the list of boundary primary keys
 */
@API(API.Status.EXPERIMENTAL)
@Nonnull
public RecordCursor<Tuple> getPrimaryKeyBoundaries(@Nonnull Tuple low, @Nonnull Tuple high) {
    final Transaction transaction = ensureContextActive();
    byte[] rangeStart = recordsSubspace().pack(low);
    byte[] rangeEnd = recordsSubspace().pack(high);
    CloseableAsyncIterator<byte[]> cursor = context.getDatabase().getLocalityProvider().getBoundaryKeys(transaction, rangeStart, rangeEnd);
    final boolean hasSplitRecordSuffix = hasSplitRecordSuffix();
    DistinctFilterCursorClosure closure = new DistinctFilterCursorClosure();
    return RecordCursor.flatMapPipelined(ignore -> RecordCursor.fromIterator(getExecutor(), cursor), (result, ignore) -> RecordCursor.fromIterator(getExecutor(), transaction.snapshot().getRange(result, rangeEnd, 1).iterator()), null, DEFAULT_PIPELINE_SIZE).map(keyValue -> {
        Tuple recordKey = recordsSubspace().unpack(keyValue.getKey());
        return hasSplitRecordSuffix ? recordKey.popBack() : recordKey;
    }).filter(closure::pred);
}

Example 14 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method rebuildIndex.

@Nonnull
// Resource usage for indexBuilder is too complicated for rule.
@SuppressWarnings("squid:S2095")
public CompletableFuture<Void> rebuildIndex(@Nonnull final Index index, @Nullable final Collection<RecordType> recordTypes, @Nonnull RebuildIndexReason reason) {
    final boolean newStore = reason == RebuildIndexReason.NEW_STORE;
    if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
        final KeyValueLogMessage msg = KeyValueLogMessage.build("rebuilding index", LogMessageKeys.INDEX_NAME, index.getName(), LogMessageKeys.INDEX_VERSION, index.getLastModifiedVersion(), LogMessageKeys.REASON, reason.name(), subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.SUBSPACE_KEY, index.getSubspaceKey());
        if (newStore) {
            LOGGER.debug(msg.toString());
        } else {
            LOGGER.info(msg.toString());
        }
    }
    long startTime = System.nanoTime();
    OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(this).setIndex(index).setRecordTypes(recordTypes).build();
    CompletableFuture<Void> future = indexBuilder.rebuildIndexAsync(this).thenCompose(vignore -> markIndexReadable(index)).handle((b, t) -> {
        if (t != null) {
            logExceptionAsWarn(KeyValueLogMessage.build("rebuilding index failed", LogMessageKeys.INDEX_NAME, index.getName(), LogMessageKeys.INDEX_VERSION, index.getLastModifiedVersion(), LogMessageKeys.REASON, reason.name(), subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.SUBSPACE_KEY, index.getSubspaceKey()), t);
        }
        // Only call method that builds in the current transaction, so never any pending work,
        // so it would work to close before returning future, which would look better to SonarQube.
        // But this is better if close ever does more.
        indexBuilder.close();
        return null;
    });
    return context.instrument(FDBStoreTimer.Events.REBUILD_INDEX, future, startTime);
}

Example 15 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method markIndexNotReadable.

@Nonnull
private CompletableFuture<Boolean> markIndexNotReadable(@Nonnull String indexName, @Nonnull IndexState indexState) {
    if (recordStoreStateRef.get() == null) {
        return preloadRecordStoreStateAsync().thenCompose(vignore -> markIndexNotReadable(indexName, indexState));
    }
    addIndexStateReadConflict(indexName);
    beginRecordStoreStateWrite();
    boolean haveFuture = false;
    try {
        // A read is done before the write in order to avoid having unnecessary
        // updates cause spurious not_committed errors.
        byte[] indexKey = indexStateSubspace().pack(indexName);
        Transaction tr = context.ensureActive();
        CompletableFuture<Boolean> future = tr.get(indexKey).thenCompose(previous -> {
            if (previous == null) {
                RangeSet indexRangeSet = new RangeSet(indexRangeSubspace(getRecordMetaData().getIndex(indexName)));
                return indexRangeSet.isEmpty(tr).thenCompose(isEmpty -> {
                    if (isEmpty) {
                        // that the index was completely built (if the range set was empty, i.e., cleared)
                        return indexRangeSet.insertRange(tr, null, null);
                    } else {
                        return AsyncUtil.READY_FALSE;
                    }
                }).thenApply(ignore -> {
                    updateIndexState(indexName, indexKey, indexState);
                    return true;
                });
            } else if (!Tuple.fromBytes(previous).get(0).equals(indexState.code())) {
                updateIndexState(indexName, indexKey, indexState);
                return AsyncUtil.READY_TRUE;
            } else {
                return AsyncUtil.READY_FALSE;
            }
        }).whenComplete((b, t) -> endRecordStoreStateWrite());
        haveFuture = true;
        return future;
    } finally {
        if (!haveFuture) {
            endRecordStoreStateWrite();
        }
    }
}