use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.
the class IndexingCommon method fillTargetIndexers.
private void fillTargetIndexers(@Nonnull List<Index> targetIndexes, @Nullable Collection<RecordType> recordTypes) {
boolean presetTypes = false;
if (recordTypes != null) {
if (targetIndexes.size() > 1) {
throw new IndexingBase.ValidationException("Can't use preset record types with multi target indexing");
}
presetTypes = true;
}
if (recordStoreBuilder.getMetaDataProvider() == null) {
throw new MetaDataException("record store builder must include metadata");
}
final RecordMetaData metaData = recordStoreBuilder.getMetaDataProvider().getRecordMetaData();
for (Index targetIndex : targetIndexes) {
Collection<RecordType> types;
if (presetTypes) {
types = recordTypes;
} else {
types = metaData.recordTypesForIndex(targetIndex);
}
boolean isSynthetic = false;
if (types.stream().anyMatch(RecordType::isSynthetic)) {
types = new SyntheticRecordPlanner(metaData, new RecordStoreState(null, null)).storedRecordTypesForIndex(targetIndex, types);
isSynthetic = true;
}
targetIndexContexts.add(new IndexContext(targetIndex, types, isSynthetic));
allRecordTypes.addAll(types);
}
}
use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.
the class IndexingScrubMissing method scrubRecordsRangeOnly.
@Nonnull
private CompletableFuture<Boolean> scrubRecordsRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
// return false when done
Index index = common.getIndex();
final RecordMetaData metaData = store.getRecordMetaData();
final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
throw new MetaDataException("Store does not have the same metadata");
}
final IndexMaintainer maintainer = store.getIndexMaintainer(index);
// scrubbing only readable, VALUE, idempotence indexes (at least for now)
validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
RangeSet rangeSet = new RangeSet(indexScrubRecordsRangeSubspace(store, index));
AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
getLimit() + 1);
final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
return ranges.onHasNext().thenCompose(hasNext -> {
if (Boolean.FALSE.equals(hasNext)) {
// Here: no more missing ranges - all done
// To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
Transaction tr = store.getContext().ensureActive();
tr.clear(indexScrubRecordsRangeSubspace(store, index).range());
return AsyncUtil.READY_FALSE;
}
final Range range = ranges.next();
final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
final AtomicBoolean hasMore = new AtomicBoolean(true);
final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
// Note that currently we only scrub idempotent indexes
final boolean isIdempotent = true;
return iterateRangeOnly(store, cursor, this::getRecordIfMissingIndex, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
if (scanLimit > 0) {
scanCounter += recordsScanned.get();
if (scanLimit <= scanCounter) {
return false;
}
}
return !Objects.equals(cont, rangeEnd);
}));
});
}
use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.
the class ValueIndexMaintainer method evaluateAggregateFunction.
@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull final IsolationLevel isolationLevel) {
final boolean reverse;
if (function.getName().equals(FunctionNames.MIN)) {
reverse = false;
} else if (function.getName().equals(FunctionNames.MAX)) {
reverse = true;
} else {
throw new MetaDataException("do not index aggregate function: " + function);
}
final int totalSize = function.getOperand().getColumnSize();
final int groupSize = totalSize - (function.getOperand() instanceof GroupingKeyExpression ? ((GroupingKeyExpression) function.getOperand()).getGroupedCount() : 1);
return scan(IndexScanType.BY_VALUE, range, null, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(isolationLevel).build(), reverse)).first().thenApply(kvo -> kvo.map(kv -> TupleHelpers.subTuple(kv.getKey(), groupSize, totalSize)).orElse(null));
}
use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.
the class VersionIndexMaintainer method updateOneKeyAsync.
// Called by updateIndexKeys in StandardIndexMaintainer.
@Override
protected <M extends Message> CompletableFuture<Void> updateOneKeyAsync(@Nonnull final FDBIndexableRecord<M> savedRecord, final boolean remove, @Nonnull final IndexEntry indexEntry) {
if (state.index.isUnique()) {
throw new MetaDataException(String.format("%s index %s does not support unique indexes", state.index.getType(), state.index.getName()));
}
final Tuple valueKey = indexEntry.getKey();
final Tuple value = indexEntry.getValue();
final long startTime = System.nanoTime();
final Tuple entryKey = indexEntryKey(valueKey, savedRecord.getPrimaryKey());
final boolean hasIncomplete = entryKey.hasIncompleteVersionstamp();
final byte[] keyBytes;
if (hasIncomplete) {
keyBytes = state.indexSubspace.packWithVersionstamp(entryKey);
} else {
keyBytes = state.indexSubspace.pack(entryKey);
}
if (remove) {
if (hasIncomplete) {
state.context.removeVersionMutation(keyBytes);
} else {
state.transaction.clear(state.indexSubspace.pack(entryKey));
}
if (state.store.getTimer() != null) {
state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.DELETE_INDEX_ENTRY, startTime);
}
} else {
final byte[] valueBytes = value.pack();
checkKeyValueSizes(savedRecord, valueKey, value, keyBytes, valueBytes);
if (hasIncomplete) {
state.context.addVersionMutation(MutationType.SET_VERSIONSTAMPED_KEY, keyBytes, valueBytes);
} else {
state.transaction.set(keyBytes, valueBytes);
}
if (state.store.getTimer() != null) {
state.store.getTimer().recordSinceNanoTime(FDBStoreTimer.Events.SAVE_INDEX_ENTRY, startTime);
}
}
return AsyncUtil.DONE;
}
use of com.apple.foundationdb.record.metadata.MetaDataException in project fdb-record-layer by FoundationDB.
the class BitmapValueIndexMaintainer method evaluateAggregateFunction.
@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull IsolationLevel isolationveLevel) {
if (!function.getName().equals(AGGREGATE_FUNCTION_NAME)) {
throw new MetaDataException("this index does not support aggregate function: " + function);
}
final RecordCursor<IndexEntry> cursor = scan(IndexScanType.BY_GROUP, range, null, new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationveLevel).build()));
final int groupPrefixSize = getGroupingCount();
long startPosition = 0;
if (range.getLow() != null && range.getLow().size() > groupPrefixSize) {
startPosition = range.getLow().getLong(groupPrefixSize);
}
int size = entrySize;
if (range.getHigh() != null && range.getHigh().size() > groupPrefixSize) {
long endPosition = range.getHigh().getLong(groupPrefixSize);
if (size > endPosition - startPosition) {
// Narrow size to what can actually be passed through from scan.
size = (int) (endPosition - startPosition);
}
}
return cursor.reduce(new BitmapAggregator(startPosition, size), (combined, kv) -> combined.append(kv.getKey().getLong(kv.getKeySize() - 1), kv.getValue().getBytes(0))).thenApply(combined -> Tuple.from(combined.asByteArray()));
}
Aggregations