Search in sources :

Example 6 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testChangeIndexDefinition.

public void testChangeIndexDefinition(boolean withCount, final String originalIndexFieldName, final String newIndexFieldName) throws Exception {
    final FDBRecordStoreBase.UserVersionChecker alwaysDisabled = new FDBRecordStoreBase.UserVersionChecker() {

        @Override
        public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
            return CompletableFuture.completedFuture(Integer.valueOf(1));
        }

        @Override
        public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
            return IndexState.DISABLED;
        }
    };
    final FDBRecordStoreBase.UserVersionChecker alwaysEnabled = new FDBRecordStoreBase.UserVersionChecker() {

        @Override
        public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
            return CompletableFuture.completedFuture(Integer.valueOf(1));
        }

        @Override
        public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
            return IndexState.READABLE;
        }
    };
    try (FDBRecordContext context = openContext()) {
        final RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestNoIndexesProto.getDescriptor());
        if (withCount) {
            builder.addUniversalIndex(COUNT_INDEX);
        }
        recordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(builder).setKeySpacePath(path).setUserVersionChecker(alwaysEnabled).createOrOpen();
        assertTrue(recordStore.getRecordStoreState().isReadable(COUNT_INDEX.getName()));
        TestNoIndexesProto.MySimpleRecord recordA = TestNoIndexesProto.MySimpleRecord.newBuilder().setNumValue(3).setStrValue("boo").build();
        recordStore.saveRecord(recordA);
        context.commit();
    }
    final Index originalIndex = new Index("index", originalIndexFieldName);
    originalIndex.setSubspaceKey(1);
    final Index newIndex = new Index("index", newIndexFieldName);
    newIndex.setSubspaceKey(2);
    final String recordType = "MySimpleRecord";
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestNoIndexesProto.getDescriptor());
        if (withCount) {
            builder.addUniversalIndex(COUNT_INDEX);
        }
        builder.addIndex(recordType, originalIndex);
        recordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(builder).setKeySpacePath(path).setUserVersionChecker(alwaysDisabled).createOrOpen();
        assertEquals(ImmutableSet.of("index"), recordStore.getRecordStoreState().getDisabledIndexNames());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestNoIndexesProto.getDescriptor());
        if (withCount) {
            builder.addUniversalIndex(COUNT_INDEX);
        }
        builder.addIndex(recordType, originalIndex);
        builder.removeIndex(originalIndex.getName());
        builder.addIndex(recordType, newIndex);
        recordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(builder).setKeySpacePath(path).setUserVersionChecker(alwaysDisabled).createOrOpen();
        assertEquals(ImmutableSet.of("index"), recordStore.getRecordStoreState().getDisabledIndexNames());
        // the index is already disabled, and we do not start enabling it during checkVersion
        assertFalse(recordStore.checkVersion(alwaysEnabled, FDBRecordStoreBase.StoreExistenceCheck.NONE).get());
        assertEquals(ImmutableSet.of("index"), recordStore.getRecordStoreState().getDisabledIndexNames());
        context.commit();
    }
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) TestNoIndexesProto(com.apple.foundationdb.record.TestNoIndexesProto) Index(com.apple.foundationdb.record.metadata.Index)

Example 7 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method noMaintenanceFilteredIndexOnCheckVersion.

@Test
public void noMaintenanceFilteredIndexOnCheckVersion() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openAnyRecordStore(TestRecordsIndexFilteringProto.getDescriptor(), context);
        TestRecordsIndexFilteringProto.MyBasicRecord recordA = TestRecordsIndexFilteringProto.MyBasicRecord.newBuilder().setRecNo(1001).setNumValue2(101).build();
        recordStore.saveRecord(recordA);
        context.commit();
        Collection<StoreTimer.Event> events = context.getTimer().getEvents();
        assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
        assertFalse(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
    }
    // add index
    RecordMetaDataHook hook = metaDataBuilder -> {
        metaDataBuilder.setVersion(metaDataBuilder.getVersion() + 1);
        metaDataBuilder.addIndex("MyBasicRecord", "value2$filtered", field("num_value_2"));
    };
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder metaData = RecordMetaData.newBuilder().setRecords(TestRecordsIndexFilteringProto.getDescriptor());
        if (hook != null) {
            hook.apply(metaData);
        }
        IndexMaintenanceFilter noneFilter = (i, r) -> IndexMaintenanceFilter.IndexValues.NONE;
        FDBRecordStore fdbRecordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(metaData).setKeySpacePath(path).setIndexMaintenanceFilter(noneFilter).uncheckedOpen();
        context.getTimer().reset();
        FDBRecordStoreBase.UserVersionChecker userVersionChecker = new FDBRecordStoreBase.UserVersionChecker() {

            @Override
            public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
                return CompletableFuture.completedFuture(Integer.valueOf(1));
            }

            @Override
            public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
                return IndexState.READABLE;
            }
        };
        fdbRecordStore.checkVersion(userVersionChecker, FDBRecordStoreBase.StoreExistenceCheck.NONE).get();
        Collection<StoreTimer.Event> events = context.getTimer().getEvents();
        assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
        assertTrue(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
    }
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) IndexScanType(com.apple.foundationdb.record.IndexScanType) Pair(org.apache.commons.lang3.tuple.Pair) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Set(java.util.Set) FanType(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType) TupleRange(com.apple.foundationdb.record.TupleRange) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) InvalidIndexEntry(com.apple.foundationdb.record.provider.foundationdb.indexes.InvalidIndexEntry) AutoContinuingCursor(com.apple.foundationdb.record.cursors.AutoContinuingCursor) Matchers.is(org.hamcrest.Matchers.is) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) ArrayList(java.util.ArrayList) Strings(com.google.common.base.Strings) CloseableAsyncIterator(com.apple.foundationdb.async.CloseableAsyncIterator) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) FDBRecordStoreBase.indexEntryKey(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.indexEntryKey) Nullable(javax.annotation.Nullable) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Tags(com.apple.test.Tags) TestRecords1EvolvedProto(com.apple.foundationdb.record.TestRecords1EvolvedProto) ExecutionException(java.util.concurrent.ExecutionException) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) ThenKeyExpression(com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression) IndexEntry(com.apple.foundationdb.record.IndexEntry) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) Random(java.util.Random) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) ImmutableSet(com.google.common.collect.ImmutableSet) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Collection(java.util.Collection) CompletionException(java.util.concurrent.CompletionException) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) TypeSafeMatcher(org.hamcrest.TypeSafeMatcher) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) TestNoIndexesProto(com.apple.foundationdb.record.TestNoIndexesProto) LazyCursor(com.apple.foundationdb.record.cursors.LazyCursor) EnumSource(org.junit.jupiter.params.provider.EnumSource) CompletableFuture(java.util.concurrent.CompletableFuture) Iterators(com.google.common.collect.Iterators) Key(com.apple.foundationdb.record.metadata.Key) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) HashSet(java.util.HashSet) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) BooleanSource(com.apple.test.BooleanSource) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) Matchers.hasEntry(org.hamcrest.Matchers.hasEntry) Description(org.hamcrest.Description) Matchers.oneOf(org.hamcrest.Matchers.oneOf) Logger(org.slf4j.Logger) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) IndexState(com.apple.foundationdb.record.IndexState) TestRecordsIndexFilteringProto(com.apple.foundationdb.record.TestRecordsIndexFilteringProto) Message(com.google.protobuf.Message) Collections(java.util.Collections) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) Index(com.apple.foundationdb.record.metadata.Index) TestRecordsIndexFilteringProto(com.apple.foundationdb.record.TestRecordsIndexFilteringProto) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 8 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class IndexingScrubDangling method scrubIndexRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubIndexRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubIndexRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubIndexRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, tupleRange, null, IndexOrphanBehavior.RETURN, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        return iterateRangeOnly(store, cursor, this::deleteIndexIfDangling, lastResult, hasMore, recordsScanned, true).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RangeSet(com.apple.foundationdb.async.RangeSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) Transaction(com.apple.foundationdb.Transaction) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) ScanProperties(com.apple.foundationdb.record.ScanProperties) IndexBuildProto(com.apple.foundationdb.record.IndexBuildProto) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) IndexState(com.apple.foundationdb.record.IndexState) TupleRange(com.apple.foundationdb.record.TupleRange) Objects(java.util.Objects) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) API(com.apple.foundationdb.annotation.API) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) AtomicReference(java.util.concurrent.atomic.AtomicReference) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Transaction(com.apple.foundationdb.Transaction) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Aggregations

RecordMetaDataProvider (com.apple.foundationdb.record.RecordMetaDataProvider)8 Index (com.apple.foundationdb.record.metadata.Index)7 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)4 RecordMetaDataBuilder (com.apple.foundationdb.record.RecordMetaDataBuilder)4 MetaDataException (com.apple.foundationdb.record.metadata.MetaDataException)4 Message (com.google.protobuf.Message)4 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)4 Range (com.apple.foundationdb.Range)3 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)3 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)3 IndexEntry (com.apple.foundationdb.record.IndexEntry)3 IndexState (com.apple.foundationdb.record.IndexState)3 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)3 ScanProperties (com.apple.foundationdb.record.ScanProperties)3 TestNoIndexesProto (com.apple.foundationdb.record.TestNoIndexesProto)3 TupleRange (com.apple.foundationdb.record.TupleRange)3 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)3 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)3 Tuple (com.apple.foundationdb.tuple.Tuple)3 Arrays (java.util.Arrays)3