Search in sources :

Example 6 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method scan.

@Nonnull
@Override
public RecordCursor<IndexEntry> scan(@Nonnull IndexScanType scanType, @Nonnull TupleRange range, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (scanType != IndexScanType.BY_GROUP) {
        throw new RecordCoreException("Can only scan bitmap index by group.");
    }
    final int groupPrefixSize = getGroupingCount();
    final long startPosition;
    if (range.getLow() != null && range.getLow().size() > groupPrefixSize && range.getLow().get(groupPrefixSize) != null) {
        if (range.getLowEndpoint() == EndpointType.RANGE_EXCLUSIVE) {
            startPosition = range.getLow().getLong(groupPrefixSize) + 1;
        } else {
            startPosition = range.getLow().getLong(groupPrefixSize);
        }
        if (startPosition % entrySize != 0) {
            range = new TupleRange(range.getLow().popBack().add(startPosition - Math.floorMod(startPosition, (long) entrySize)), range.getHigh(), EndpointType.RANGE_INCLUSIVE, range.getHighEndpoint());
        }
    } else {
        startPosition = Long.MIN_VALUE;
    }
    final long endPosition;
    if (range.getHigh() != null && range.getHigh().size() > groupPrefixSize && range.getHigh().get(groupPrefixSize) != null) {
        if (range.getHighEndpoint() == EndpointType.RANGE_INCLUSIVE) {
            endPosition = range.getHigh().getLong(groupPrefixSize) + 1;
        } else {
            endPosition = range.getHigh().getLong(groupPrefixSize);
        }
        if (endPosition % entrySize != 0) {
            range = new TupleRange(range.getLow(), range.getHigh().popBack().add(endPosition + Math.floorMod(entrySize - endPosition, (long) entrySize)), range.getLowEndpoint(), EndpointType.RANGE_INCLUSIVE);
        }
    } else {
        endPosition = Long.MAX_VALUE;
    }
    return scan(range, continuation, scanProperties).map(indexEntry -> {
        final long entryStart = indexEntry.getKey().getLong(groupPrefixSize);
        final byte[] entryBitmap = indexEntry.getValue().getBytes(0);
        final long entryEnd = entryStart + entryBitmap.length * 8;
        if (entryStart < startPosition || entryEnd > endPosition) {
            final long trimmedStart = Math.max(entryStart, startPosition);
            final long trimmedEnd = Math.min(entryEnd, endPosition);
            if (trimmedStart < trimmedEnd) {
                final Tuple trimmedKey = indexEntry.getKey().popBack().add(trimmedStart);
                final byte[] trimmedBitmap = new byte[((int) (trimmedEnd - trimmedStart) + 7) / 8];
                for (long i = trimmedStart; i < trimmedEnd; i++) {
                    int offset = (int) (i - entryStart);
                    if ((entryBitmap[offset / 8] & (byte) (1 << (offset % 8))) != 0) {
                        int trimmedOffset = (int) (i - trimmedStart);
                        trimmedBitmap[trimmedOffset / 8] |= (byte) (1 << (trimmedOffset % 8));
                    }
                }
                final Tuple subValue = Tuple.from(trimmedBitmap);
                return Optional.of(new IndexEntry(indexEntry.getIndex(), trimmedKey, subValue));
            } else {
                return Optional.<IndexEntry>empty();
            }
        } else {
            return Optional.of(indexEntry);
        }
    }).filter(Optional::isPresent).map(Optional::get);
}
Also used : RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Optional(java.util.Optional) IndexEntry(com.apple.foundationdb.record.IndexEntry) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 7 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexMaintainer method evaluateAggregateFunction.

@Override
@Nonnull
public CompletableFuture<Tuple> evaluateAggregateFunction(@Nonnull IndexAggregateFunction function, @Nonnull TupleRange range, @Nonnull IsolationLevel isolationveLevel) {
    if (!function.getName().equals(AGGREGATE_FUNCTION_NAME)) {
        throw new MetaDataException("this index does not support aggregate function: " + function);
    }
    final RecordCursor<IndexEntry> cursor = scan(IndexScanType.BY_GROUP, range, null, new ScanProperties(ExecuteProperties.newBuilder().setIsolationLevel(isolationveLevel).build()));
    final int groupPrefixSize = getGroupingCount();
    long startPosition = 0;
    if (range.getLow() != null && range.getLow().size() > groupPrefixSize) {
        startPosition = range.getLow().getLong(groupPrefixSize);
    }
    int size = entrySize;
    if (range.getHigh() != null && range.getHigh().size() > groupPrefixSize) {
        long endPosition = range.getHigh().getLong(groupPrefixSize);
        if (size > endPosition - startPosition) {
            // Narrow size to what can actually be passed through from scan.
            size = (int) (endPosition - startPosition);
        }
    }
    return cursor.reduce(new BitmapAggregator(startPosition, size), (combined, kv) -> combined.append(kv.getKey().getLong(kv.getKeySize() - 1), kv.getValue().getBytes(0))).thenApply(combined -> Tuple.from(combined.asByteArray()));
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) FunctionNames(com.apple.foundationdb.record.FunctionNames) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) IndexMaintainerState(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ByteBuffer(java.nio.ByteBuffer) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) MutationType(com.apple.foundationdb.MutationType) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) EndpointType(com.apple.foundationdb.record.EndpointType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) FDBIndexableRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexableRecord) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) IndexFunctionHelper(com.apple.foundationdb.record.provider.foundationdb.IndexFunctionHelper) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) TupleRange(com.apple.foundationdb.record.TupleRange) List(java.util.List) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexEntry(com.apple.foundationdb.record.IndexEntry) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Nonnull(javax.annotation.Nonnull)

Example 8 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildEndpoints.

/**
 * Builds (transactionally) the endpoints of an index. What this means is that builds everything from the beginning of
 * the key space to the first record and everything from the last record to the end of the key space.
 * There won't be any records within these ranges (except for the last record of the record store), but
 * it does mean that any records in the future that get added to these ranges will correctly update
 * the index. This means, e.g., that if the workload primarily adds records to the record store
 * after the current last record (because perhaps the primary key is based off of an atomic counter
 * or the current time), running this method will be highly contentious, but once it completes,
 * the rest of the index build should happen without any more conflicts.
 *
 * This will return a (possibly null) {@link TupleRange} that contains the primary keys of the
 * first and last records within the record store. This can then be used to either build the
 * range right away or to then divy-up the remaining ranges between multiple agents working
 * in parallel if one desires.
 *
 * @param store the record store in which to rebuild the index
 * @param recordsScanned continues counter
 * @return a future that will contain the range of records in the interior of the record store
 */
@Nonnull
public CompletableFuture<TupleRange> buildEndpoints(@Nonnull FDBRecordStore store, @Nullable AtomicLong recordsScanned) {
    final RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(common.getIndex()));
    if (TupleRange.ALL.equals(recordsRange)) {
        return buildEndpoints(store, rangeSet, recordsScanned);
    }
    // If records do not occupy whole range, first mark outside as built.
    final Range asRange = recordsRange.toRange();
    return CompletableFuture.allOf(rangeSet.insertRange(store.ensureContextActive(), null, asRange.begin), rangeSet.insertRange(store.ensureContextActive(), asRange.end, null)).thenCompose(vignore -> buildEndpoints(store, rangeSet, recordsScanned));
}
Also used : RangeSet(com.apple.foundationdb.async.RangeSet) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) Nonnull(javax.annotation.Nonnull)

Example 9 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class IndexingByRecords method buildEndpoints.

@Nonnull
private CompletableFuture<TupleRange> buildEndpoints(@Nonnull FDBRecordStore store, @Nonnull RangeSet rangeSet, @Nullable AtomicLong recordsScanned) {
    boolean isIdempotent = store.getIndexMaintainer(common.getIndex()).isIdempotent();
    final IsolationLevel isolationLevel = isIdempotent ? // before this method's query) will be re-indexed.
    IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties limit1 = ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(isolationLevel).build();
    final ScanProperties forward = new ScanProperties(limit1);
    RecordCursor<FDBStoredRecord<Message>> beginCursor = store.scanRecords(recordsRange, null, forward);
    CompletableFuture<Tuple> begin = beginCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple firstTuple = result.get().getPrimaryKey();
            return buildRange(store, null, firstTuple, recordsScanned).thenApply(vignore -> firstTuple);
        } else {
            // Empty range -- add the whole thing.
            return rangeSet.insertRange(store.ensureContextActive(), null, null).thenApply(bignore -> null);
        }
    });
    final ScanProperties backward = new ScanProperties(limit1, true);
    RecordCursor<FDBStoredRecord<Message>> endCursor = store.scanRecords(recordsRange, null, backward);
    CompletableFuture<Tuple> end = endCursor.onNext().thenCompose(result -> {
        if (result.hasNext()) {
            Tuple lastTuple = result.get().getPrimaryKey();
            return buildRange(store, lastTuple, null, recordsScanned).thenApply(vignore -> lastTuple);
        } else {
            // by the above future, so this has nothing to do.
            return CompletableFuture.completedFuture(null);
        }
    });
    return begin.thenCombine(end, (firstTuple, lastTuple) -> {
        if (firstTuple == null || firstTuple.equals(lastTuple)) {
            return null;
        } else {
            return new TupleRange(firstTuple, lastTuple, EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
        }
    });
}
Also used : ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 10 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreBase method scanIndexRecordsBetween.

/**
 * Scan the records pointed to by an index between two indexed values.
 * @param indexName the name of the index
 * @param low the low value for the first indexed field
 * @param high the high value for the first indexed field
 * @return a cursor that return records pointed to by the index
 */
@Nonnull
default RecordCursor<FDBIndexedRecord<M>> scanIndexRecordsBetween(@Nonnull final String indexName, @Nullable final Object low, @Nullable final Object high) {
    final Tuple lowTuple = Tuple.from(low);
    final Tuple highTuple = Tuple.from(high);
    final TupleRange range = new TupleRange(lowTuple, highTuple, EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE);
    return scanIndexRecords(indexName, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN);
}
Also used : TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Aggregations

TupleRange (com.apple.foundationdb.record.TupleRange)48 Tuple (com.apple.foundationdb.tuple.Tuple)29 Nonnull (javax.annotation.Nonnull)28 Message (com.google.protobuf.Message)21 Index (com.apple.foundationdb.record.metadata.Index)16 Test (org.junit.jupiter.api.Test)16 ScanProperties (com.apple.foundationdb.record.ScanProperties)15 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)14 Nullable (javax.annotation.Nullable)14 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)11 IndexEntry (com.apple.foundationdb.record.IndexEntry)11 List (java.util.List)11 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)10 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)10 RecordCursor (com.apple.foundationdb.record.RecordCursor)10 IndexScanType (com.apple.foundationdb.record.IndexScanType)9 CompletableFuture (java.util.concurrent.CompletableFuture)9 Range (com.apple.foundationdb.Range)8 API (com.apple.foundationdb.annotation.API)8 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)8