Search in sources :

Example 1 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class IndexingCommon method fillTargetIndexers.

private void fillTargetIndexers(@Nonnull List<Index> targetIndexes, @Nullable Collection<RecordType> recordTypes) {
    boolean presetTypes = false;
    if (recordTypes != null) {
        if (targetIndexes.size() > 1) {
            throw new IndexingBase.ValidationException("Can't use preset record types with multi target indexing");
        }
        presetTypes = true;
    }
    if (recordStoreBuilder.getMetaDataProvider() == null) {
        throw new MetaDataException("record store builder must include metadata");
    }
    final RecordMetaData metaData = recordStoreBuilder.getMetaDataProvider().getRecordMetaData();
    for (Index targetIndex : targetIndexes) {
        Collection<RecordType> types;
        if (presetTypes) {
            types = recordTypes;
        } else {
            types = metaData.recordTypesForIndex(targetIndex);
        }
        boolean isSynthetic = false;
        if (types.stream().anyMatch(RecordType::isSynthetic)) {
            types = new SyntheticRecordPlanner(metaData, new RecordStoreState(null, null)).storedRecordTypesForIndex(targetIndex, types);
            isSynthetic = true;
        }
        targetIndexContexts.add(new IndexContext(targetIndex, types, isSynthetic));
        allRecordTypes.addAll(types);
    }
}
Also used : RecordMetaData(com.apple.foundationdb.record.RecordMetaData) RecordType(com.apple.foundationdb.record.metadata.RecordType) SyntheticRecordPlanner(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner) Index(com.apple.foundationdb.record.metadata.Index) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException)

Example 2 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method scrubRecordsRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubRecordsRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubRecordsRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubRecordsRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        // Note that currently we only scrub idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfMissingIndex, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) IndexState(com.apple.foundationdb.record.IndexState) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) API(com.apple.foundationdb.annotation.API) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Transaction(com.apple.foundationdb.Transaction) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 3 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class ValueIndexMaintainer method evaluateAggregateFunction.

@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull final IsolationLevel isolationLevel) {
    final boolean reverse;
    if (function.getName().equals(FunctionNames.MIN)) {
        reverse = false;
    } else if (function.getName().equals(FunctionNames.MAX)) {
        reverse = true;
    } else {
        throw new MetaDataException("do not index aggregate function: " + function);
    }
    final int totalSize = function.getOperand().getColumnSize();
    final int groupSize = totalSize - (function.getOperand() instanceof GroupingKeyExpression ? ((GroupingKeyExpression) function.getOperand()).getGroupedCount() : 1);
    return scan(IndexScanType.BY_VALUE, range, null, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(isolationLevel).build(), reverse)).first().thenApply(kvo -> kvo.map(kv -> TupleHelpers.subTuple(kv.getKey(), groupSize, totalSize)).orElse(null));
}
Also used : GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) ScanProperties(com.apple.foundationdb.record.ScanProperties) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Nonnull(javax.annotation.Nonnull)

Example 4 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class VersionIndexMaintainer method updateOneKeyAsync.

// Called by updateIndexKeys in StandardIndexMaintainer.
@Override
protected <M extends Message> CompletableFuture<Void> updateOneKeyAsync(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final IndexEntry indexEntry) {
    if (state.index.isUnique()) {
        throw new MetaDataException(String.format("%s index %s does not support unique indexes", state.index.getType(), state.index.getName()));
    }
    final Tuple valueKey = indexEntry.getKey();
    final Tuple value = indexEntry.getValue();
    final long startTime = System.nanoTime();
    final Tuple entryKey = indexEntryKey(valueKey, savedRecord.getPrimaryKey());
    final boolean hasIncomplete = entryKey.hasIncompleteVersionstamp();
    final byte[] keyBytes;
    if (hasIncomplete) {
        keyBytes = state.indexSubspace.packWithVersionstamp(entryKey);
    } else {
        keyBytes = state.indexSubspace.pack(entryKey);
    }
    if (remove) {
        if (hasIncomplete) {
            state.context.removeVersionMutation(keyBytes);
        } else {
            state.transaction.clear(state.indexSubspace.pack(entryKey));
        }
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.DELETE_INDEX_ENTRY, startTime);
        }
    } else {
        final byte[] valueBytes = value.pack();
        checkKeyValueSizes(savedRecord, valueKey, value, keyBytes, valueBytes);
        if (hasIncomplete) {
            state.context.addVersionMutation(MutationType.SET_VERSIONSTAMPED_KEY, keyBytes, valueBytes);
        } else {
            state.transaction.set(keyBytes, valueBytes);
        }
        if (state.store.getTimer() != null) {
            state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.SAVE_INDEX_ENTRY, startTime);
        }
    }
    return AsyncUtil.DONE;
}
Also used : MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Tuple(com.apple.foundationdb.tuple.Tuple)

Example 5 with MetaDataException

use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method evaluateAggregateFunction.

@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull IsolationLevel isolationveLevel) {
    if (!function.getName().equals(AGGREGATE_FUNCTION_NAME)) {
        throw new MetaDataException("this index does not support aggregate function: " + function);
    }
    final RecordCursor<IndexEntry> cursor = scan(IndexScanType.BY_GROUP, range, null, new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationveLevel).build()));
    final int groupPrefixSize = getGroupingCount();
    long startPosition = 0;
    if (range.getLow() != null && range.getLow().size() > groupPrefixSize) {
        startPosition = range.getLow().getLong(groupPrefixSize);
    }
    int size = entrySize;
    if (range.getHigh() != null && range.getHigh().size() > groupPrefixSize) {
        long endPosition = range.getHigh().getLong(groupPrefixSize);
        if (size > endPosition - startPosition) {
            // Narrow size to what can actually be passed through from scan.
            size = (int) (endPosition - startPosition);
        }
    }
    return cursor.reduce(new BitmapAggregator(startPosition, size), (combined, kv) -> combined.append(kv.getKey().getLong(kv.getKeySize() - 1), kv.getValue().getBytes(0))).thenApply(combined -> Tuple.from(combined.asByteArray()));
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) FunctionNames(com.apple.foundationdb.record.FunctionNames) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) IndexMaintainerState(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ByteBuffer(java.nio.ByteBuffer) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) MutationType(com.apple.foundationdb.MutationType) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) EndpointType(com.apple.foundationdb.record.EndpointType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBIndexableRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexableRecord) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) IndexFunctionHelper(com.apple.foundationdb.record.provider.foundationdb.IndexFunctionHelper) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) TupleRange(com.apple.foundationdb.record.TupleRange) List(java.util.List) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexEntry(com.apple.foundationdb.record.IndexEntry) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Nonnull(javax.annotation.Nonnull)

Aggregations

MetaDataException (com.apple.foundationdb.record.metadata.MetaDataException)61 Test (org.junit.jupiter.api.Test)33 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)29 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)28 MetaDataProtoTest (com.apple.foundationdb.record.metadata.MetaDataProtoTest)25 Nonnull (javax.annotation.Nonnull)24 Index (com.apple.foundationdb.record.metadata.Index)22 Descriptors (com.google.protobuf.Descriptors)14 Tuple (com.apple.foundationdb.tuple.Tuple)13 List (java.util.List)12 ScanProperties (com.apple.foundationdb.record.ScanProperties)11 TupleRange (com.apple.foundationdb.record.TupleRange)11 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)11 IndexEntry (com.apple.foundationdb.record.IndexEntry)10 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)10 DescriptorProtos (com.google.protobuf.DescriptorProtos)10 TestRecords1Proto (com.apple.foundationdb.record.TestRecords1Proto)9 Message (com.google.protobuf.Message)9 Collectors (java.util.stream.Collectors)9 IndexScanType (com.apple.foundationdb.record.IndexScanType)8