Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 16 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method updateSyntheticIndexes.

@API(API.Status.EXPERIMENTAL)
private <M extends Message> void updateSyntheticIndexes(@Nullable FDBStoredRecord<M> oldRecord, @Nullable FDBStoredRecord<M> newRecord, @Nonnull final List<CompletableFuture<Void>> futures) {
    final SyntheticRecordPlanner planner = new SyntheticRecordPlanner(this);
    // Index maintainers are not required to be thread-safe, so only do one synthetic record at a time.
    final int pipelineSize = 1;
    if (oldRecord != null && newRecord != null && oldRecord.getRecordType() == newRecord.getRecordType()) {
        // TODO: An important optimization here is determining that no field used in the join condition or
        // indexed in the synthetic record is changed, in which case all this can be skipped.
        final SyntheticRecordFromStoredRecordPlan plan = planner.fromStoredType(newRecord.getRecordType(), true);
        if (plan == null) {
            return;
        }
        final Map<RecordType, Collection<IndexMaintainer>> maintainers = getSyntheticMaintainers(plan.getSyntheticRecordTypes());
        final Map<Tuple, FDBSyntheticRecord> oldRecords = new ConcurrentHashMap<>();
        CompletableFuture<Void> future = plan.execute(this, oldRecord).forEach(syntheticRecord -> oldRecords.put(syntheticRecord.getPrimaryKey(), syntheticRecord));
        // @SpotBugsSuppressWarnings("NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE", justification = "https://github.com/spotbugs/spotbugs/issues/552")
        @Nonnull final FDBStoredRecord<M> theNewRecord = newRecord;
        future = future.thenCompose(v -> plan.execute(this, theNewRecord).forEachAsync(syntheticRecord -> runSyntheticMaintainers(maintainers, oldRecords.remove(syntheticRecord.getPrimaryKey()), syntheticRecord), pipelineSize));
        future = future.thenCompose(v -> {
            // Any synthetic record that was generated by the plan on the old record but not by the plan on the new record needs to be removed from its indexes.
            final List<CompletableFuture<Void>> subFutures = new ArrayList<>();
            for (FDBSyntheticRecord oldSyntheticRecord : oldRecords.values()) {
                CompletableFuture<Void> subFuture = runSyntheticMaintainers(maintainers, oldSyntheticRecord, null);
                if (!MoreAsyncUtil.isCompletedNormally(subFuture)) {
                    subFutures.add(subFuture);
                }
            }
            if (subFutures.isEmpty()) {
                return AsyncUtil.DONE;
            } else if (subFutures.size() == 1) {
                return subFutures.get(0);
            } else {
                return AsyncUtil.whenAll(subFutures);
            }
        });
        futures.add(future);
    } else {
        if (oldRecord != null) {
            final SyntheticRecordFromStoredRecordPlan plan = planner.fromStoredType(oldRecord.getRecordType(), true);
            if (plan != null) {
                final Map<RecordType, Collection<IndexMaintainer>> maintainers = getSyntheticMaintainers(plan.getSyntheticRecordTypes());
                futures.add(plan.execute(this, oldRecord).forEachAsync(syntheticRecord -> runSyntheticMaintainers(maintainers, syntheticRecord, null), pipelineSize));
            }
        }
        if (newRecord != null) {
            final SyntheticRecordFromStoredRecordPlan plan = planner.fromStoredType(newRecord.getRecordType(), true);
            if (plan != null) {
                final Map<RecordType, Collection<IndexMaintainer>> maintainers = getSyntheticMaintainers(plan.getSyntheticRecordTypes());
                futures.add(plan.execute(this, newRecord).forEachAsync(syntheticRecord -> runSyntheticMaintainers(maintainers, null, syntheticRecord), pipelineSize));
            }
        }
    }
}

Example 17 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method markIndexReadable.

/**
 * Marks an index as readable. See the version of
 * {@link #markIndexReadable(String) markIndexReadable()}
 * that takes a {@link String} as a parameter for more details.
 *
 * @param index the index to mark readable
 * @return a future that will either complete exceptionally if the index can not
 * be made readable or will contain <code>true</code> if the store was modified
 * and <code>false</code> otherwise
 */
@Nonnull
public CompletableFuture<Boolean> markIndexReadable(@Nonnull Index index) {
    if (recordStoreStateRef.get() == null) {
        return preloadRecordStoreStateAsync().thenCompose(vignore -> markIndexReadable(index));
    }
    addIndexStateReadConflict(index.getName());
    beginRecordStoreStateWrite();
    boolean haveFuture = false;
    try {
        Transaction tr = ensureContextActive();
        byte[] indexKey = indexStateSubspace().pack(index.getName());
        CompletableFuture<Boolean> future = tr.get(indexKey).thenCompose(previous -> {
            if (previous != null) {
                CompletableFuture<Optional<Range>> builtFuture = firstUnbuiltRange(index);
                CompletableFuture<Optional<RecordIndexUniquenessViolation>> uniquenessFuture = whenAllIndexUniquenessCommitChecks(index).thenCompose(vignore -> scanUniquenessViolations(index, 1).first());
                return CompletableFuture.allOf(builtFuture, uniquenessFuture).thenApply(vignore -> {
                    Optional<Range> firstUnbuilt = context.join(builtFuture);
                    Optional<RecordIndexUniquenessViolation> uniquenessViolation = context.join(uniquenessFuture);
                    if (firstUnbuilt.isPresent()) {
                        throw new IndexNotBuiltException("Attempted to make unbuilt index readable", firstUnbuilt.get(), LogMessageKeys.INDEX_NAME, index.getName(), "unbuiltRangeBegin", ByteArrayUtil2.loggable(firstUnbuilt.get().begin), "unbuiltRangeEnd", ByteArrayUtil2.loggable(firstUnbuilt.get().end), subspaceProvider.logKey(), subspaceProvider.toString(context), LogMessageKeys.SUBSPACE_KEY, index.getSubspaceKey());
                    } else if (uniquenessViolation.isPresent()) {
                        RecordIndexUniquenessViolation wrapped = new RecordIndexUniquenessViolation("Uniqueness violation when making index readable", uniquenessViolation.get());
                        wrapped.addLogInfo(LogMessageKeys.INDEX_NAME, index.getName(), subspaceProvider.logKey(), subspaceProvider.toString(context));
                        throw wrapped;
                    } else {
                        updateIndexState(index.getName(), indexKey, IndexState.READABLE);
                        clearReadableIndexBuildData(tr, index);
                        return true;
                    }
                });
            } else {
                return AsyncUtil.READY_FALSE;
            }
        }).whenComplete((b, t) -> endRecordStoreStateWrite()).thenApply(this::addRemoveReplacedIndexesCommitCheckIfChanged);
        haveFuture = true;
        return future;
    } finally {
        if (!haveFuture) {
            endRecordStoreStateWrite();
        }
    }
}

Example 18 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method checkRebuildIndexes.

private CompletableFuture<Void> checkRebuildIndexes(@Nullable UserVersionChecker userVersionChecker, @Nonnull RecordMetaDataProto.DataStoreInfo.Builder info, int oldFormatVersion, @Nonnull RecordMetaData metaData, int oldMetaDataVersion, boolean rebuildRecordCounts, List<CompletableFuture<Void>> work) {
    final boolean newStore = oldFormatVersion == 0;
    final Map<Index, List<RecordType>> indexes = metaData.getIndexesToBuildSince(oldMetaDataVersion);
    if (!indexes.isEmpty()) {
        // If all the new indexes are only for a record type whose primary key has a type prefix, then we can scan less.
        RecordType singleRecordTypeWithPrefixKey = singleRecordTypeWithPrefixKey(indexes);
        final AtomicLong recordCountRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordCount = getAndRememberFutureLong(recordCountRef, () -> getRecordCountForRebuildIndexes(newStore, rebuildRecordCounts, indexes, singleRecordTypeWithPrefixKey));
        AtomicLong recordsSizeRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordsSize = getAndRememberFutureLong(recordsSizeRef, () -> getRecordSizeForRebuildIndexes(singleRecordTypeWithPrefixKey));
        if (singleRecordTypeWithPrefixKey == null && formatVersion >= SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION && omitUnsplitRecordSuffix) {
            // Check to see if the unsplit format can be upgraded on an empty store.
            // Only works if singleRecordTypeWithPrefixKey is null as otherwise, the recordCount will not contain
            // all records
            work.add(lazyRecordCount.get().thenAccept(recordCount -> {
                if (recordCount == 0) {
                    if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
                        KeyValueLogMessage msg = KeyValueLogMessage.build("upgrading unsplit format on empty store", LogMessageKeys.NEW_FORMAT_VERSION, formatVersion, subspaceProvider.logKey(), subspaceProvider.toString(context));
                        if (newStore) {
                            LOGGER.debug(msg.toString());
                        } else {
                            LOGGER.info(msg.toString());
                        }
                    }
                    omitUnsplitRecordSuffix = formatVersion < SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION;
                    info.clearOmitUnsplitRecordSuffix();
                    // We used snapshot to determine emptiness, and are now acting on it.
                    addRecordsReadConflict();
                }
            }));
        }
        Map<Index, CompletableFuture<IndexState>> newStates = getStatesForRebuildIndexes(userVersionChecker, indexes, lazyRecordCount, lazyRecordsSize, newStore, oldMetaDataVersion, oldFormatVersion);
        return rebuildIndexes(indexes, newStates, work, newStore ? RebuildIndexReason.NEW_STORE : RebuildIndexReason.FEW_RECORDS, oldMetaDataVersion).thenRun(() -> {
            // Log after checking all index states
            maybeLogIndexesNeedingRebuilding(newStates, recordCountRef, recordsSizeRef, rebuildRecordCounts, newStore);
            context.increment(FDBStoreTimer.Counts.INDEXES_NEED_REBUILDING, newStates.entrySet().size());
        });
    } else {
        return work.isEmpty() ? AsyncUtil.DONE : AsyncUtil.whenAll(work);
    }
}

Example 19 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class IndexingBase method buildIndexAsync.

// buildIndexAsync - the main indexing function. Builds and commits indexes asynchronously; throttling to avoid overloading the system.
public CompletableFuture<Void> buildIndexAsync(boolean markReadable) {
    KeyValueLogMessage message = KeyValueLogMessage.build("build index online", LogMessageKeys.SHOULD_MARK_READABLE, markReadable).addKeysAndValues(indexingLogMessageKeyValues()).addKeysAndValues(common.indexLogMessageKeyValues());
    final CompletableFuture<Void> buildIndexAsyncFuture;
    FDBDatabaseRunner runner = common.getRunner();
    Index index = common.getPrimaryIndex();
    if (common.isUseSynchronizedSession()) {
        buildIndexAsyncFuture = runner.runAsync(context -> openRecordStore(context).thenApply(store -> indexBuildLockSubspace(store, index)), common.indexLogMessageKeyValues("IndexingBase::indexBuildLockSubspace")).thenCompose(lockSubspace -> runner.startSynchronizedSessionAsync(lockSubspace, common.getLeaseLengthMillis())).thenCompose(synchronizedRunner -> {
            message.addKeyAndValue(LogMessageKeys.SESSION_ID, synchronizedRunner.getSessionId());
            return runWithSynchronizedRunnerAndEndSession(synchronizedRunner, () -> handleStateAndDoBuildIndexAsync(markReadable, message));
        });
    } else {
        message.addKeyAndValue(LogMessageKeys.SESSION_ID, "none");
        common.setSynchronizedSessionRunner(null);
        buildIndexAsyncFuture = handleStateAndDoBuildIndexAsync(markReadable, message);
    }
    return buildIndexAsyncFuture.whenComplete((vignore, ex) -> {
        if (LOGGER.isWarnEnabled() && (ex != null)) {
            message.addKeyAndValue(LogMessageKeys.RESULT, "failure");
            LOGGER.warn(message.toString(), ex);
        } else if (LOGGER.isInfoEnabled()) {
            message.addKeyAndValue(LogMessageKeys.RESULT, "success");
            LOGGER.info(message.toString());
        }
    });
}

Example 20 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class IndexingBase method rebuildIndexAsync.

// rebuildIndexAsync - builds the whole index inline (without committing)
@Nonnull
public CompletableFuture<Void> rebuildIndexAsync(@Nonnull FDBRecordStore store) {
    return forEachTargetIndex(index -> store.clearAndMarkIndexWriteOnly(index).thenCompose(bignore -> {
        // Insert the full range into the range set. (The internal rebuild method only indexes the records and
        // does not update the range set.) This is important because if marking the index as readable fails (for
        // example, because of uniqueness violations), we still want to record in the range set that the entire
        // range was built so that future index builds don't re-scan the record data and so that non-idempotent
        // indexes know to update the index on all record saves.
        Transaction tr = store.ensureContextActive();
        RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(index));
        return rangeSet.insertRange(tr, null, null);
    })).thenCompose(vignore -> rebuildIndexInternalAsync(store));
}