Search in sources :

Example 26 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method countRecords.

@Override
@Nonnull
public CompletableFuture<Integer> countRecords(@Nullable Tuple low, @Nullable Tuple high, @Nonnull EndpointType lowEndpoint, @Nonnull EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final Subspace recordsSubspace = recordsSubspace();
    RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setContinuation(continuation).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
    if (getRecordMetaData().isSplitLongRecords()) {
        return new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), null, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearRowAndTimeLimits))).getCount();
    } else {
        return keyValues.getCount();
    }
}
Also used : ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyValue(com.apple.foundationdb.KeyValue) Subspace(com.apple.foundationdb.subspace.Subspace) CursorLimitManager(com.apple.foundationdb.record.cursors.CursorLimitManager) Nonnull(javax.annotation.Nonnull)

Example 27 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingByIndex method rebuildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> rebuildRangeOnly(@Nonnull FDBRecordStore store, Tuple cont, @Nonnull AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // idempotence - We could have verified it at the first iteration only, but the repeating checks seem harmless
    validateOrThrowEx(maintainer.isIdempotent(), "target index is not idempotent");
    // readability - This method shouldn't block if one has already opened the record store (as we did)
    final Index srcIndex = getSourceIndex(store.getRecordMetaData());
    validateOrThrowEx(store.isIndexReadable(srcIndex), "source index is not readable");
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final TupleRange tupleRange = TupleRange.between(cont, null);
    RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(srcIndex.getName(), IndexScanType.BY_VALUE, tupleRange, null, scanProperties);
    final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    // Note that currently indexing by index is online implemented for idempotent indexes
    final boolean isIdempotent = true;
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : null);
}
Also used : Message(com.google.protobuf.Message) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Nonnull(javax.annotation.Nonnull)

Example 28 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingByIndex method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    validateSameMetadataOrThrow(store);
    final Index index = common.getIndex();
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // idempotence - We could have verified it at the first iteration only, but the repeating checks seem harmless
    validateOrThrowEx(maintainer.isIdempotent(), "target index is not idempotent");
    // readability - This method shouldn't block if one has already opened the record store (as we did)
    Index srcIndex = getSourceIndex(store.getRecordMetaData());
    validateOrThrowEx(store.isIndexReadable(srcIndex), "source index is not readable");
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(srcIndex.getName(), IndexScanType.BY_VALUE, tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        // Note that currently indexing by index is online implemented for idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}
Also used : RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Collection(java.util.Collection) TupleRange(com.apple.foundationdb.record.TupleRange) ByteString(com.google.protobuf.ByteString) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) API(com.apple.foundationdb.annotation.API) Message(com.google.protobuf.Message) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 29 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingMultiTargetByRecords method buildRangeOnly.

@Nonnull
private CompletableFuture<Boolean> buildRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    /* Multi target consistency:
         * 1. Identify missing ranges from only the first index
         * 2. Update all indexes' range sets as the indexes are built - each inserted range is validated as empty.
         * 3. While each index as readable, we validate that its range is completely built.
         */
    validateSameMetadataOrThrow(store);
    RangeSet rangeSet = new RangeSet(store.indexRangeSubspace(common.getPrimaryIndex()));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final List<Index> targetIndexes = common.getTargetIndexes();
    final List<RangeSet> targetRangeSets = targetIndexes.stream().map(targetIndex -> new RangeSet(store.indexRangeSubspace(targetIndex))).collect(Collectors.toList());
    final boolean isIdempotent = areTheyAllIdempotent(store, targetIndexes);
    final IsolationLevel isolationLevel = isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel).setReturnedRowLimit(// always respect limit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // no more missing ranges - all done
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> insertRanges(store.ensureContextActive(), targetRangeSets, packOrNull(rangeStart), packOrNull(cont)).thenApply(ignore -> !Objects.equals(cont, rangeEnd)));
    });
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) API(com.apple.foundationdb.annotation.API) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Message(com.google.protobuf.Message) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 30 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class IndexingMultiTargetByRecords method rebuildRangeOnly.

@Nonnull
private CompletableFuture<Tuple> rebuildRangeOnly(@Nonnull FDBRecordStore store, Tuple cont, @Nonnull AtomicLong recordsScanned) {
    validateSameMetadataOrThrow(store);
    final boolean isIdempotent = areTheyAllIdempotent(store, common.getTargetIndexes());
    final IsolationLevel isolationLevel = isIdempotent ? IsolationLevel.SNAPSHOT : IsolationLevel.SERIALIZABLE;
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(isolationLevel);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    final TupleRange tupleRange = TupleRange.between(cont, null);
    RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
    final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
    final AtomicBoolean hasMore = new AtomicBoolean(true);
    return iterateRangeOnly(store, cursor, this::getRecordIfTypeMatch, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : null);
}
Also used : IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Message(com.google.protobuf.Message) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Nonnull(javax.annotation.Nonnull)

Aggregations

ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)38 Nonnull (javax.annotation.Nonnull)29 Message (com.google.protobuf.Message)23 RecordCursor (com.apple.foundationdb.record.RecordCursor)20 ScanProperties (com.apple.foundationdb.record.ScanProperties)18 List (java.util.List)17 Index (com.apple.foundationdb.record.metadata.Index)15 FDBQueriedRecord (com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord)15 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)15 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)15 RecordCursorResult (com.apple.foundationdb.record.RecordCursorResult)14 Function (java.util.function.Function)14 Nullable (javax.annotation.Nullable)13 API (com.apple.foundationdb.annotation.API)12 EvaluationContext (com.apple.foundationdb.record.EvaluationContext)12 Tuple (com.apple.foundationdb.tuple.Tuple)12 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)11 TupleRange (com.apple.foundationdb.record.TupleRange)11 ArrayList (java.util.ArrayList)11 Collections (java.util.Collections)10