Examples with IndexState com.apple.foundationdb.record.IndexState used on opensource projects

Example 1 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class IndexingThrottle method throttledRunAsync.

@Nonnull
<R> CompletableFuture<R> throttledRunAsync(@Nonnull final Function<FDBRecordStore, CompletableFuture<R>> function, @Nonnull final BiFunction<R, Throwable, Pair<R, Throwable>> handlePostTransaction, @Nullable final BiConsumer<FDBException, List<Object>> handleLessenWork, @Nullable final List<Object> additionalLogMessageKeyValues) {
    List<Object> onlineIndexerLogMessageKeyValues = new ArrayList<>(Arrays.asList(LogMessageKeys.INDEX_NAME, common.getTargetIndexesNames(), LogMessageKeys.INDEXER_ID, common.getUuid()));
    if (additionalLogMessageKeyValues != null) {
        onlineIndexerLogMessageKeyValues.addAll(additionalLogMessageKeyValues);
    }
    AtomicInteger tries = new AtomicInteger(0);
    CompletableFuture<R> ret = new CompletableFuture<>();
    AtomicLong toWait = new AtomicLong(common.getRunner().getDatabase().getFactory().getInitialDelayMillis());
    AsyncUtil.whileTrue(() -> {
        loadConfig();
        return common.getRunner().runAsync(context -> common.getRecordStoreBuilder().copyBuilder().setContext(context).openAsync().thenCompose(store -> {
            for (Index index : common.getTargetIndexes()) {
                IndexState indexState = store.getIndexState(index);
                if (indexState != expectedIndexState) {
                    throw new RecordCoreStorageException("Unexpected index state", LogMessageKeys.INDEX_NAME, index.getName(), common.getRecordStoreBuilder().getSubspaceProvider().logKey(), common.getRecordStoreBuilder().getSubspaceProvider().toString(context), LogMessageKeys.INDEX_STATE, indexState, LogMessageKeys.INDEX_STATE_PRECONDITION, expectedIndexState);
                }
            }
            return function.apply(store);
        }), handlePostTransaction, onlineIndexerLogMessageKeyValues).handle((value, e) -> {
            if (e == null) {
                ret.complete(value);
                return AsyncUtil.READY_FALSE;
            } else {
                int currTries = tries.getAndIncrement();
                FDBException fdbE = getFDBException(e);
                if (currTries < common.config.getMaxRetries() && fdbE != null && lessenWorkCodes.contains(fdbE.getCode())) {
                    if (handleLessenWork != null) {
                        handleLessenWork.accept(fdbE, onlineIndexerLogMessageKeyValues);
                    }
                    long delay = (long) (Math.random() * toWait.get());
                    toWait.set(Math.min(toWait.get() * 2, common.getRunner().getDatabase().getFactory().getMaxDelayMillis()));
                    if (LOGGER.isWarnEnabled()) {
                        final KeyValueLogMessage message = KeyValueLogMessage.build("Retrying Runner Exception", LogMessageKeys.INDEXER_CURR_RETRY, currTries, LogMessageKeys.INDEXER_MAX_RETRIES, common.config.getMaxRetries(), LogMessageKeys.DELAY, delay, LogMessageKeys.LIMIT, limit);
                        message.addKeysAndValues(onlineIndexerLogMessageKeyValues);
                        LOGGER.warn(message.toString(), e);
                    }
                    return MoreAsyncUtil.delayedFuture(delay, TimeUnit.MILLISECONDS).thenApply(vignore3 -> true);
                } else {
                    return completeExceptionally(ret, e, onlineIndexerLogMessageKeyValues);
                }
            }
        }).thenCompose(Function.identity());
    }, common.getRunner().getExecutor()).whenComplete((vignore, e) -> {
        if (e != null) {
            // Just update ret and ignore the returned future.
            completeExceptionally(ret, e, onlineIndexerLogMessageKeyValues);
        }
    });
    return ret;
}

Example 2 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method loadIndexStatesAsync.

@Nonnull
private CompletableFuture<Map<String, IndexState>> loadIndexStatesAsync(@Nonnull IsolationLevel isolationLevel) {
    Subspace isSubspace = getSubspace().subspace(Tuple.from(INDEX_STATE_SPACE_KEY));
    KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(isSubspace).setContext(getContext()).setRange(TupleRange.ALL).setContinuation(null).setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel).setDefaultCursorStreamingMode(CursorStreamingMode.WANT_ALL).build())).build();
    FDBStoreTimer timer = getTimer();
    CompletableFuture<Map<String, IndexState>> result = cursor.asList().thenApply(list -> {
        Map<String, IndexState> indexStateMap;
        if (list.isEmpty()) {
            indexStateMap = Collections.emptyMap();
        } else {
            ImmutableMap.Builder<String, IndexState> indexStateMapBuilder = ImmutableMap.builder();
            for (KeyValue kv : list) {
                String indexName = isSubspace.unpack(kv.getKey()).getString(0);
                Object code = Tuple.fromBytes(kv.getValue()).get(0);
                indexStateMapBuilder.put(indexName, IndexState.fromCode(code));
                if (timer != null) {
                    timer.increment(FDBStoreTimer.Counts.LOAD_STORE_STATE_KEY);
                    timer.increment(FDBStoreTimer.Counts.LOAD_STORE_STATE_KEY_BYTES, kv.getKey().length);
                    timer.increment(FDBStoreTimer.Counts.LOAD_STORE_STATE_VALUE_BYTES, kv.getValue().length);
                }
            }
            indexStateMap = indexStateMapBuilder.build();
        }
        return indexStateMap;
    });
    if (timer != null) {
        result = timer.instrument(FDBStoreTimer.Events.LOAD_RECORD_STORE_INDEX_META_DATA, result, context.getExecutor());
    }
    return result;
}

Example 3 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method markIndexNotReadable.

@Nonnull
private CompletableFuture<Boolean> markIndexNotReadable(@Nonnull String indexName, @Nonnull IndexState indexState) {
    if (recordStoreStateRef.get() == null) {
        return preloadRecordStoreStateAsync().thenCompose(vignore -> markIndexNotReadable(indexName, indexState));
    }
    addIndexStateReadConflict(indexName);
    beginRecordStoreStateWrite();
    boolean haveFuture = false;
    try {
        // A read is done before the write in order to avoid having unnecessary
        // updates cause spurious not_committed errors.
        byte[] indexKey = indexStateSubspace().pack(indexName);
        Transaction tr = context.ensureActive();
        CompletableFuture<Boolean> future = tr.get(indexKey).thenCompose(previous -> {
            if (previous == null) {
                RangeSet indexRangeSet = new RangeSet(indexRangeSubspace(getRecordMetaData().getIndex(indexName)));
                return indexRangeSet.isEmpty(tr).thenCompose(isEmpty -> {
                    if (isEmpty) {
                        // that the index was completely built (if the range set was empty, i.e., cleared)
                        return indexRangeSet.insertRange(tr, null, null);
                    } else {
                        return AsyncUtil.READY_FALSE;
                    }
                }).thenApply(ignore -> {
                    updateIndexState(indexName, indexKey, indexState);
                    return true;
                });
            } else if (!Tuple.fromBytes(previous).get(0).equals(indexState.code())) {
                updateIndexState(indexName, indexKey, indexState);
                return AsyncUtil.READY_TRUE;
            } else {
                return AsyncUtil.READY_FALSE;
            }
        }).whenComplete((b, t) -> endRecordStoreStateWrite());
        haveFuture = true;
        return future;
    } finally {
        if (!haveFuture) {
            endRecordStoreStateWrite();
        }
    }
}

Example 4 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method checkRebuildIndexes.

private CompletableFuture<Void> checkRebuildIndexes(@Nullable UserVersionChecker userVersionChecker, @Nonnull RecordMetaDataProto.DataStoreInfo.Builder info, int oldFormatVersion, @Nonnull RecordMetaData metaData, int oldMetaDataVersion, boolean rebuildRecordCounts, List<CompletableFuture<Void>> work) {
    final boolean newStore = oldFormatVersion == 0;
    final Map<Index, List<RecordType>> indexes = metaData.getIndexesToBuildSince(oldMetaDataVersion);
    if (!indexes.isEmpty()) {
        // If all the new indexes are only for a record type whose primary key has a type prefix, then we can scan less.
        RecordType singleRecordTypeWithPrefixKey = singleRecordTypeWithPrefixKey(indexes);
        final AtomicLong recordCountRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordCount = getAndRememberFutureLong(recordCountRef, () -> getRecordCountForRebuildIndexes(newStore, rebuildRecordCounts, indexes, singleRecordTypeWithPrefixKey));
        AtomicLong recordsSizeRef = new AtomicLong(-1);
        final Supplier<CompletableFuture<Long>> lazyRecordsSize = getAndRememberFutureLong(recordsSizeRef, () -> getRecordSizeForRebuildIndexes(singleRecordTypeWithPrefixKey));
        if (singleRecordTypeWithPrefixKey == null && formatVersion >= SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION && omitUnsplitRecordSuffix) {
            // Check to see if the unsplit format can be upgraded on an empty store.
            // Only works if singleRecordTypeWithPrefixKey is null as otherwise, the recordCount will not contain
            // all records
            work.add(lazyRecordCount.get().thenAccept(recordCount -> {
                if (recordCount == 0) {
                    if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
                        KeyValueLogMessage msg = KeyValueLogMessage.build("upgrading unsplit format on empty store", LogMessageKeys.NEW_FORMAT_VERSION, formatVersion, subspaceProvider.logKey(), subspaceProvider.toString(context));
                        if (newStore) {
                            LOGGER.debug(msg.toString());
                        } else {
                            LOGGER.info(msg.toString());
                        }
                    }
                    omitUnsplitRecordSuffix = formatVersion < SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION;
                    info.clearOmitUnsplitRecordSuffix();
                    // We used snapshot to determine emptiness, and are now acting on it.
                    addRecordsReadConflict();
                }
            }));
        }
        Map<Index, CompletableFuture<IndexState>> newStates = getStatesForRebuildIndexes(userVersionChecker, indexes, lazyRecordCount, lazyRecordsSize, newStore, oldMetaDataVersion, oldFormatVersion);
        return rebuildIndexes(indexes, newStates, work, newStore ? RebuildIndexReason.NEW_STORE : RebuildIndexReason.FEW_RECORDS, oldMetaDataVersion).thenRun(() -> {
            // Log after checking all index states
            maybeLogIndexesNeedingRebuilding(newStates, recordCountRef, recordsSizeRef, rebuildRecordCounts, newStore);
            context.increment(FDBStoreTimer.Counts.INDEXES_NEED_REBUILDING, newStates.entrySet().size());
        });
    } else {
        return work.isEmpty() ? AsyncUtil.DONE : AsyncUtil.whenAll(work);
    }
}

Example 5 with IndexState

use of com.apple.foundationdb.record.IndexState in project fdb-record-layer by FoundationDB.

the class IndexBuildState method loadIndexBuildStateAsync.

/**
 * Load the build progress ({@link IndexBuildState}) of the given index in the given record store asynchronously.
 * @param store the record store containing the index
 * @param index the index needed to be checked
 * @return a future that completes to the index build state
 */
@Nonnull
public static CompletableFuture<IndexBuildState> loadIndexBuildStateAsync(FDBRecordStoreBase<?> store, Index index) {
    IndexState indexState = store.getUntypedRecordStore().getIndexState(index);
    if (indexState != IndexState.WRITE_ONLY) {
        return CompletableFuture.completedFuture(new IndexBuildState(indexState));
    }
    CompletableFuture<Long> recordsInTotalFuture;
    try {
        recordsInTotalFuture = store.getSnapshotRecordCount();
    } catch (AggregateFunctionNotSupportedException ex) {
        // getSnapshotRecordCount failed, very likely it is because there is no suitable COUNT type index
        // defined.
        recordsInTotalFuture = CompletableFuture.completedFuture(null);
    }
    return loadRecordsScannedAsync(store, index).thenCombine(recordsInTotalFuture, (scannedRecords, recordsInTotal) -> new IndexBuildState(indexState, scannedRecords, recordsInTotal));
}