Examples with RecordMetaDataProvider com.apple.foundationdb.record.RecordMetaDataProvider used on opensource projects

Example 1 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class IndexingScrubMissing method scrubRecordsRangeOnly.

@Nonnull
private CompletableFuture<Boolean> scrubRecordsRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
    // return false when done
    Index index = common.getIndex();
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
    final IndexMaintainer maintainer = store.getIndexMaintainer(index);
    // scrubbing only readable, VALUE, idempotence indexes (at least for now)
    validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
    validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
    validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
    RangeSet rangeSet = new RangeSet(indexScrubRecordsRangeSubspace(store, index));
    AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
    final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
    getLimit() + 1);
    final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
    return ranges.onHasNext().thenCompose(hasNext -> {
        if (Boolean.FALSE.equals(hasNext)) {
            // Here: no more missing ranges - all done
            // To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
            Transaction tr = store.getContext().ensureActive();
            tr.clear(indexScrubRecordsRangeSubspace(store, index).range());
            return AsyncUtil.READY_FALSE;
        }
        final Range range = ranges.next();
        final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
        final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
        final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
        final RecordCursor<FDBStoredRecord<Message>> cursor = store.scanRecords(tupleRange, null, scanProperties);
        final AtomicBoolean hasMore = new AtomicBoolean(true);
        final AtomicReference<RecordCursorResult<FDBStoredRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
        final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
        // Note that currently we only scrub idempotent indexes
        final boolean isIdempotent = true;
        return iterateRangeOnly(store, cursor, this::getRecordIfMissingIndex, lastResult, hasMore, recordsScanned, isIdempotent).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getPrimaryKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
            if (scanLimit > 0) {
                scanCounter += recordsScanned.get();
                if (scanLimit <= scanCounter) {
                    return false;
                }
            }
            return !Objects.equals(cont, rangeEnd);
        }));
    });
}

Example 2 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class IndexingBase method validateSameMetadataOrThrow.

protected void validateSameMetadataOrThrow(FDBRecordStore store) {
    final RecordMetaData metaData = store.getRecordMetaData();
    final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
    if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
        throw new MetaDataException("Store does not have the same metadata");
    }
}

Example 3 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreSplitRecordsTest method clearOmitUnsplitRecordSuffixOverlapping.

@Test
public void clearOmitUnsplitRecordSuffixOverlapping() {
    final RecordMetaDataBuilder metaData = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
    FDBRecordStore.Builder builder = FDBRecordStore.newBuilder().setKeySpacePath(path).setMetaDataProvider(metaData).setFormatVersion(FDBRecordStore.FORMAT_CONTROL_FORMAT_VERSION);
    try (FDBRecordContext context = openContext()) {
        recordStore = builder.setContext(context).create();
        commit(context);
    }
    FDBRecordContext context1 = openContext();
    FDBRecordStore recordStore = builder.setContext(context1).open();
    TestRecords1Proto.MySimpleRecord record = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1L).setStrValueIndexed("abc").build();
    FDBStoredRecord<Message> saved = recordStore.saveRecord(record);
    metaData.addIndex("MySimpleRecord", "num_value_2");
    builder.setFormatVersion(FDBRecordStore.MAX_SUPPORTED_FORMAT_VERSION);
    // If we did build, we'd create a read confict on the new index.
    // We want to test that a conflict comes from the clearing itself.
    final FDBRecordStoreBase.UserVersionChecker dontBuild = new FDBRecordStoreBase.UserVersionChecker() {

        @Override
        public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
            return CompletableFuture.completedFuture(0);
        }

        @Override
        public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
            return IndexState.DISABLED;
        }
    };
    builder.setUserVersionChecker(dontBuild);
    FDBRecordContext context2 = openContext();
    FDBRecordStore recordStore2 = builder.setContext(context2).open();
    assertFalse(recordStore2.getRecordStoreState().getStoreHeader().getOmitUnsplitRecordSuffix());
    commit(context1);
    assertThrows(FDBExceptions.FDBStoreTransactionConflictException.class, () -> commit(context2));
    try (FDBRecordContext context = openContext()) {
        recordStore = builder.setContext(context).open();
        FDBStoredRecord<Message> loaded = recordStore.loadRecord(saved.getPrimaryKey());
        assertNotNull(loaded);
        assertEquals(saved.getRecord(), loaded.getRecord());
        assertTrue(recordStore.getRecordStoreState().getStoreHeader().getOmitUnsplitRecordSuffix());
    }
}

Example 4 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreCountRecordsTest method recountAndClearRecords.

private void recountAndClearRecords(boolean useIndex) throws Exception {
    final CountMetaDataHook countMetaDataHook = new CountMetaDataHook();
    countMetaDataHook.baseHook = metaData -> metaData.removeIndex(COUNT_INDEX.getName());
    final int startingPoint = 7890;
    final int value1 = 12345;
    final int value2 = 54321;
    final int value3 = 24567;
    try (FDBRecordContext context = openContext()) {
        // Simulate the state the store would be in if this were done before counting was added.
        recordStore = getStoreBuilder(context, simpleMetaData(countMetaDataHook)).setFormatVersion(FDBRecordStore.INFO_ADDED_FORMAT_VERSION).uncheckedOpen();
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_EXISTS).join();
        for (int i = 0; i < 90; i++) {
            recordStore.saveRecord(makeRecord(i + startingPoint, value1, i % 5));
        }
        commit(context);
    }
    KeyExpression key3 = field("num_value_3_indexed");
    countMetaDataHook.metaDataVersion++;
    countMetaDataHook.baseHook = countKeyHook(key3, useIndex, countMetaDataHook.metaDataVersion);
    try (FDBRecordContext context = openContext()) {
        recordStore = getStoreBuilder(context, simpleMetaData(countMetaDataHook)).setFormatVersion(FDBRecordStore.RECORD_COUNT_ADDED_FORMAT_VERSION).uncheckedOpen();
        for (int i = 90; i < 100; i++) {
            recordStore.saveRecord(makeRecord(i + startingPoint, value2, i % 5));
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        recordStore = getStoreBuilder(context, simpleMetaData(countMetaDataHook)).setFormatVersion(FDBRecordStore.RECORD_COUNT_ADDED_FORMAT_VERSION).uncheckedOpen();
        assertEquals(10, recordStore.getSnapshotRecordCount().join().longValue(), "should only see new records");
        commit(context);
    }
    // Need to allow immediate rebuild of new count index.
    final FDBRecordStoreBase.UserVersionChecker alwaysEnabled = new FDBRecordStoreBase.UserVersionChecker() {

        @Override
        public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
            return CompletableFuture.completedFuture(1);
        }

        @Override
        public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
            return IndexState.READABLE;
        }
    };
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context, countMetaDataHook);
        // Index is rebuilt here automatically in useIndex case
        recordStore.checkVersion(alwaysEnabled, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        assertEquals(100, recordStore.getSnapshotRecordCount().join().longValue(), "should see all records");
        assertEquals(20, recordStore.getSnapshotRecordCount(key3, Key.Evaluated.scalar(2)).join().longValue());
        commit(context);
    }
    KeyExpression key2 = field("num_value_2");
    countMetaDataHook.metaDataVersion++;
    countMetaDataHook.baseHook = countKeyHook(key2, useIndex, countMetaDataHook.metaDataVersion);
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context, countMetaDataHook);
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        if (useIndex) {
            // Need to manually rebuild index in this case.
            Index index = recordStore.getRecordMetaData().getIndex("record_count");
            recordStore.rebuildIndex(index).get();
            assertThat(recordStore.isIndexReadable(index), is(true));
        }
        assertEquals(100, recordStore.getSnapshotRecordCount().join().longValue(), "should see all records");
        for (int i = 0; i < 32; i++) {
            recordStore.saveRecord(makeRecord(i + startingPoint + 1000, value3, 0));
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context, countMetaDataHook);
        assertEquals(90, recordStore.getSnapshotRecordCount(key2, Key.Evaluated.scalar(value1)).join().longValue());
        assertEquals(10, recordStore.getSnapshotRecordCount(key2, Key.Evaluated.scalar(value2)).join().longValue());
        assertEquals(32, recordStore.getSnapshotRecordCount(key2, Key.Evaluated.scalar(value3)).join().longValue());
    }
    KeyExpression pkey = field("rec_no");
    countMetaDataHook.metaDataVersion++;
    countMetaDataHook.baseHook = countKeyHook(pkey, useIndex, countMetaDataHook.metaDataVersion);
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context, countMetaDataHook);
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.NONE).join();
        if (useIndex) {
            // Need to manually rebuild index in this case.
            Index index = recordStore.getRecordMetaData().getIndex("record_count");
            recordStore.rebuildIndex(index).get();
            assertThat(recordStore.isIndexReadable(index), is(true));
        }
        assertEquals(132, recordStore.getSnapshotRecordCount().join().longValue());
        for (int i = 0; i < 100; i++) {
            assertEquals(1, recordStore.getSnapshotRecordCount(pkey, Key.Evaluated.scalar(i + startingPoint)).join().longValue(), "Incorrect when i is " + i);
        }
    }
}

Example 5 with RecordMetaDataProvider

use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testSelectiveIndexDisable.

@Test
public void testSelectiveIndexDisable() throws Exception {
    final FDBRecordStoreBase.UserVersionChecker selectiveEnable = new FDBRecordStoreBase.UserVersionChecker() {

        @Override
        public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
            return CompletableFuture.completedFuture(Integer.valueOf(1));
        }

        @Override
        public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
            if (index.getName().equals("index-1")) {
                return IndexState.DISABLED;
            } else {
                return IndexState.READABLE;
            }
        }
    };
    final RecordMetaDataBuilder metaData = RecordMetaData.newBuilder().setRecords(TestNoIndexesProto.getDescriptor());
    metaData.addUniversalIndex(COUNT_INDEX);
    final FDBRecordStore.Builder storeBuilder = FDBRecordStore.newBuilder().setUserVersionChecker(selectiveEnable).setMetaDataProvider(metaData);
    try (FDBRecordContext context = openContext()) {
        // builds count index
        recordStore = storeBuilder.setContext(context).setKeySpacePath(path).create();
        assertTrue(recordStore.getRecordStoreState().isReadable(COUNT_INDEX.getName()));
        TestNoIndexesProto.MySimpleRecord recordA = TestNoIndexesProto.MySimpleRecord.newBuilder().setNumValue(3).setStrValue("boo").build();
        recordStore.saveRecord(recordA);
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        metaData.addIndex("MySimpleRecord", "index-1", "num_value");
        metaData.addIndex("MySimpleRecord", "index-2", "str_value");
        // builds index-2, disables index-1
        recordStore = storeBuilder.setContext(context).setKeySpacePath(path).open();
        assertEquals(ImmutableSet.of("index-1"), recordStore.getRecordStoreState().getDisabledIndexNames());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        recordStore = storeBuilder.setContext(context).setKeySpacePath(path).open();
        recordStore.clearAndMarkIndexWriteOnly("index-1").join();
        context.commit();
    }
    try (OnlineIndexer onlineIndexBuilder = OnlineIndexer.forRecordStoreAndIndex(recordStore, "index-1")) {
        onlineIndexBuilder.buildIndex();
    }
    try (FDBRecordContext context = openContext()) {
        // does not disable anything
        recordStore = storeBuilder.setContext(context).setKeySpacePath(path).open();
        assertEquals(Collections.emptySet(), recordStore.getRecordStoreState().getDisabledIndexNames());
        context.commit();
    }
}