Examples with Subspace com.apple.foundationdb.subspace.Subspace used on opensource projects

Example 46 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method repairRecordKeys.

/**
 * Validate and repair known potential issues with record keys. Currently, this method is capable of identifying
 * and repairing the following scenarios:
 * <ul>
 *     <li>For record stores in which record splitting is disabled but the {@code omitUnsplitRecordSuffix} flag
 *         is {@code true}, keys found missing the {@link SplitHelper#UNSPLIT_RECORD} suffix will be repaired.</li>
 *     <li>For record stores in which record splitting is disabled, but the record key suffix is found to be
 *         a value other than {@link SplitHelper#UNSPLIT_RECORD}, the {@link FDBStoreTimer.Counts#INVALID_SPLIT_SUFFIX}
 *         counter will be incremented.
 *     <li>For record stores in which record splitting is disabled, but the record key is longer than expected,
 *         the {@link FDBStoreTimer.Counts#INVALID_KEY_LENGTH} counter will be incremented.
 * </ul>
 *
 * @param continuation continuation from a previous repair attempt or {@code null} to start from the beginning
 * @param scanProperties properties to provide scan limits on the repair process
 *   record keys should be logged
 * @param isDryRun if true, no repairs are made, however counters involving irregular keys or keys that would
 *   would have been repaired are incremented
 * @return a future that completes to a continuation or {@code null} if the repair has been completed
 */
@Nonnull
public CompletableFuture<byte[]> repairRecordKeys(@Nullable byte[] continuation, @Nonnull ScanProperties scanProperties, final boolean isDryRun) {
    // If the records aren't split to begin with, then there is nothing to do.
    if (getRecordMetaData().isSplitLongRecords()) {
        return CompletableFuture.completedFuture(null);
    }
    if (scanProperties.getExecuteProperties().getIsolationLevel().isSnapshot()) {
        throw new RecordCoreArgumentException("Cannot repair record key split markers at SNAPSHOT isolation level").addLogInfo(LogMessageKeys.SCAN_PROPERTIES, scanProperties).addLogInfo(subspaceProvider.logKey(), subspaceProvider.toString(context));
    }
    final Subspace recordSubspace = recordsSubspace();
    KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(recordSubspace).setContext(getRecordContext()).setContinuation(continuation).setScanProperties(scanProperties).build();
    final AtomicReference<byte[]> nextContinuation = new AtomicReference<>();
    final FDBRecordContext context = getContext();
    return AsyncUtil.whileTrue(() -> cursor.onNext().thenApply(result -> {
        if (!result.hasNext()) {
            if (result.hasStoppedBeforeEnd()) {
                nextContinuation.set(result.getContinuation().toBytes());
            }
            return false;
        }
        repairRecordKeyIfNecessary(context, recordSubspace, result.get(), isDryRun);
        return true;
    })).thenApply(ignored -> nextContinuation.get());
}

Example 47 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method deleteStore.

/**
 * Delete the record store at the given {@link KeySpacePath}. This behaves like
 * {@link #deleteStore(FDBRecordContext, Subspace)} on the record store saved
 * at {@link KeySpacePath#toSubspace(FDBRecordContext)}.
 *
 * @param context the transactional context in which to delete the record store
 * @param path the path to the record store
 * @see #deleteStore(FDBRecordContext, Subspace)
 */
public static void deleteStore(FDBRecordContext context, KeySpacePath path) {
    final Subspace subspace = path.toSubspace(context);
    deleteStore(context, subspace);
}

Example 48 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method countRecords.

@Override
@Nonnull
public CompletableFuture<Integer> countRecords(@Nullable Tuple low, @Nullable Tuple high, @Nonnull EndpointType lowEndpoint, @Nonnull EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final Subspace recordsSubspace = recordsSubspace();
    RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setContinuation(continuation).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
    if (getRecordMetaData().isSplitLongRecords()) {
        return new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), null, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearRowAndTimeLimits))).getCount();
    } else {
        return keyValues.getCount();
    }
}

Example 49 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class IndexingBase method iterateRangeOnly.

/**
 * iterate cursor's items and index them.
 *
 * @param store the record store.
 * @param cursor iteration items.
 * @param getRecordToIndex function to convert cursor's item to a record that should be indexed (or null, if inapplicable)
 * @param nextResultCont when return, if hasMore is true, holds the last cursored result - unprocessed - as a
 * continuation item.
 * @param hasMore when return, true if the cursor's source is not exhausted (not more items in range).
 * @param recordsScanned when return, number of scanned records.
 * @param isIdempotent are all the built indexes idempotent
 * @param <T> cursor result's type.
 *
 * @return hasMore, nextResultCont, and recordsScanned.
 */
protected <T> CompletableFuture<Void> iterateRangeOnly(@Nonnull FDBRecordStore store, @Nonnull RecordCursor<T> cursor, @Nonnull BiFunction<FDBRecordStore, RecordCursorResult<T>, CompletableFuture<FDBStoredRecord<Message>>> getRecordToIndex, @Nonnull AtomicReference<RecordCursorResult<T>> nextResultCont, @Nonnull AtomicBoolean hasMore, @Nullable AtomicLong recordsScanned, final boolean isIdempotent) {
    final FDBStoreTimer timer = getRunner().getTimer();
    final FDBRecordContext context = store.getContext();
    // Need to do this each transaction because other index enabled state might have changed. Could cache based on that.
    // Copying the state also guards against changes made by other online building from check version.
    AtomicLong recordsScannedCounter = new AtomicLong();
    final AtomicReference<RecordCursorResult<T>> nextResult = new AtomicReference<>(null);
    return AsyncUtil.whileTrue(() -> cursor.onNext().thenCompose(result -> {
        RecordCursorResult<T> currResult;
        final boolean isExhausted;
        if (result.hasNext()) {
            // has next, process one previous item (if exists)
            currResult = nextResult.get();
            nextResult.set(result);
            if (currResult == null) {
                // that was the first item, nothing to process
                return AsyncUtil.READY_TRUE;
            }
            isExhausted = false;
        } else {
            // end of the cursor list
            timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT);
            if (!result.getNoNextReason().isSourceExhausted()) {
                nextResultCont.set(nextResult.get());
                hasMore.set(true);
                return AsyncUtil.READY_FALSE;
            }
            // source is exhausted, fall down to handle the last item and return with hasMore=false
            currResult = nextResult.get();
            if (currResult == null) {
                // there was no data
                hasMore.set(false);
                return AsyncUtil.READY_FALSE;
            }
            // here, process the last item and return
            nextResult.set(null);
            isExhausted = true;
        }
        // here: currResult must have value
        timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED);
        recordsScannedCounter.incrementAndGet();
        return getRecordToIndex.apply(store, currResult).thenCompose(rec -> {
            if (null == rec) {
                if (isExhausted) {
                    hasMore.set(false);
                    return AsyncUtil.READY_FALSE;
                }
                return AsyncUtil.READY_TRUE;
            }
            // This record should be indexed. Add it to the transaction.
            if (isIdempotent) {
                store.addRecordReadConflict(rec.getPrimaryKey());
            }
            timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED);
            final CompletableFuture<Void> updateMaintainer = updateMaintainerBuilder(store, rec);
            if (isExhausted) {
                // we've just processed the last item
                hasMore.set(false);
                return updateMaintainer.thenApply(vignore -> false);
            }
            return updateMaintainer.thenCompose(vignore -> context.getApproximateTransactionSize().thenApply(size -> {
                if (size >= common.config.getMaxWriteLimitBytes()) {
                    // the transaction becomes too big - stop iterating
                    timerIncrement(timer, FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE);
                    nextResultCont.set(nextResult.get());
                    hasMore.set(true);
                    return false;
                }
                return true;
            }));
        });
    }), cursor.getExecutor()).thenApply(vignore -> {
        long recordsScannedInTransaction = recordsScannedCounter.get();
        if (recordsScanned != null) {
            recordsScanned.addAndGet(recordsScannedInTransaction);
        }
        if (common.isTrackProgress()) {
            for (Index index : common.getTargetIndexes()) {
                final Subspace scannedRecordsSubspace = indexBuildScannedRecordsSubspace(store, index);
                store.context.ensureActive().mutate(MutationType.ADD, scannedRecordsSubspace.getKey(), FDBRecordStore.encodeRecordCount(recordsScannedInTransaction));
            }
        }
        return null;
    });
}

Example 50 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class BunchedMapIterator method onHasNext.

@Override
public CompletableFuture<Boolean> onHasNext() {
    if (done) {
        return AsyncUtil.READY_FALSE;
    }
    if (currEntryList != null && currEntryIndex >= 0 && currEntryIndex < currEntryList.size()) {
        return AsyncUtil.READY_TRUE;
    }
    if (hasNextFuture == null) {
        hasNextFuture = underlying.onHasNext().thenCompose(doesHaveNext -> {
            if (doesHaveNext) {
                return AsyncUtil.whileTrue(() -> {
                    KeyValue underlyingNext = underlying.peek();
                    if (!subspace.contains(underlyingNext.getKey())) {
                        if (ByteArrayUtil.compareUnsigned(underlyingNext.getKey(), subspaceKey) * (reverse ? -1 : 1) < 0) {
                            // We haven't gotten to this subspace yet, so try the next key.
                            underlying.next();
                            return underlying.onHasNext();
                        } else {
                            // We are done iterating through this subspace. Return
                            // without advancing the scan to support scanning
                            // over multiple subspaces.
                            done = true;
                            return AsyncUtil.READY_FALSE;
                        }
                    }
                    // Advance the underlying scan.
                    underlying.next();
                    final K boundaryKey = bunchedMap.getSerializer().deserializeKey(underlyingNext.getKey(), subspaceKey.length);
                    List<Map.Entry<K, V>> nextEntryList = bunchedMap.getSerializer().deserializeEntries(boundaryKey, underlyingNext.getValue());
                    if (nextEntryList.isEmpty()) {
                        // No entries in list. Try next key.
                        return underlying.onHasNext();
                    }
                    int nextItemIndex = reverse ? nextEntryList.size() - 1 : 0;
                    if (!continuationSatisfied) {
                        while (nextItemIndex >= 0 && nextItemIndex < nextEntryList.size() && bunchedMap.getKeyComparator().compare(continuationKey, nextEntryList.get(nextItemIndex).getKey()) * (reverse ? -1 : 1) >= 0) {
                            nextItemIndex += reverse ? -1 : 1;
                        }
                        if (nextItemIndex < 0 || nextItemIndex >= nextEntryList.size()) {
                            // through this entry. Move on to the next key in the database.
                            return underlying.onHasNext();
                        } else {
                            continuationSatisfied = true;
                        }
                    }
                    // TODO: We can be more exact about conflict ranges here.
                    currEntryIndex = nextItemIndex;
                    currEntryList = nextEntryList;
                    return AsyncUtil.READY_FALSE;
                }, tr.getExecutor()).thenApply(vignore -> {
                    if (currEntryList == null || currEntryIndex < 0 || currEntryIndex >= currEntryList.size()) {
                        done = true;
                    }
                    return !done;
                });
            } else {
                // Exhausted scan.
                done = true;
                return AsyncUtil.READY_FALSE;
            }
        });
    }
    return hasNextFuture;
}