use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method testChangeIndexDefinition.
public void testChangeIndexDefinition(boolean withCount, final String originalIndexFieldName, final String newIndexFieldName) throws Exception {
final FDBRecordStoreBase.UserVersionChecker alwaysDisabled = new FDBRecordStoreBase.UserVersionChecker() {
@Override
public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
return CompletableFuture.completedFuture(Integer.valueOf(1));
}
@Override
public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
return IndexState.DISABLED;
}
};
final FDBRecordStoreBase.UserVersionChecker alwaysEnabled = new FDBRecordStoreBase.UserVersionChecker() {
@Override
public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
return CompletableFuture.completedFuture(Integer.valueOf(1));
}
@Override
public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
return IndexState.READABLE;
}
};
try (FDBRecordContext context = openContext()) {
final RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestNoIndexesProto.getDescriptor());
if (withCount) {
builder.addUniversalIndex(COUNT_INDEX);
}
recordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(builder).setKeySpacePath(path).setUserVersionChecker(alwaysEnabled).createOrOpen();
assertTrue(recordStore.getRecordStoreState().isReadable(COUNT_INDEX.getName()));
TestNoIndexesProto.MySimpleRecord recordA = TestNoIndexesProto.MySimpleRecord.newBuilder().setNumValue(3).setStrValue("boo").build();
recordStore.saveRecord(recordA);
context.commit();
}
final Index originalIndex = new Index("index", originalIndexFieldName);
originalIndex.setSubspaceKey(1);
final Index newIndex = new Index("index", newIndexFieldName);
newIndex.setSubspaceKey(2);
final String recordType = "MySimpleRecord";
try (FDBRecordContext context = openContext()) {
RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestNoIndexesProto.getDescriptor());
if (withCount) {
builder.addUniversalIndex(COUNT_INDEX);
}
builder.addIndex(recordType, originalIndex);
recordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(builder).setKeySpacePath(path).setUserVersionChecker(alwaysDisabled).createOrOpen();
assertEquals(ImmutableSet.of("index"), recordStore.getRecordStoreState().getDisabledIndexNames());
context.commit();
}
try (FDBRecordContext context = openContext()) {
RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestNoIndexesProto.getDescriptor());
if (withCount) {
builder.addUniversalIndex(COUNT_INDEX);
}
builder.addIndex(recordType, originalIndex);
builder.removeIndex(originalIndex.getName());
builder.addIndex(recordType, newIndex);
recordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(builder).setKeySpacePath(path).setUserVersionChecker(alwaysDisabled).createOrOpen();
assertEquals(ImmutableSet.of("index"), recordStore.getRecordStoreState().getDisabledIndexNames());
// the index is already disabled, and we do not start enabling it during checkVersion
assertFalse(recordStore.checkVersion(alwaysEnabled, FDBRecordStoreBase.StoreExistenceCheck.NONE).get());
assertEquals(ImmutableSet.of("index"), recordStore.getRecordStoreState().getDisabledIndexNames());
context.commit();
}
}
use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method noMaintenanceFilteredIndexOnCheckVersion.
@Test
public void noMaintenanceFilteredIndexOnCheckVersion() throws Exception {
try (FDBRecordContext context = openContext()) {
openAnyRecordStore(TestRecordsIndexFilteringProto.getDescriptor(), context);
TestRecordsIndexFilteringProto.MyBasicRecord recordA = TestRecordsIndexFilteringProto.MyBasicRecord.newBuilder().setRecNo(1001).setNumValue2(101).build();
recordStore.saveRecord(recordA);
context.commit();
Collection<StoreTimer.Event> events = context.getTimer().getEvents();
assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
assertFalse(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
}
// add index
RecordMetaDataHook hook = metaDataBuilder -> {
metaDataBuilder.setVersion(metaDataBuilder.getVersion() + 1);
metaDataBuilder.addIndex("MyBasicRecord", "value2$filtered", field("num_value_2"));
};
try (FDBRecordContext context = openContext()) {
RecordMetaDataBuilder metaData = RecordMetaData.newBuilder().setRecords(TestRecordsIndexFilteringProto.getDescriptor());
if (hook != null) {
hook.apply(metaData);
}
IndexMaintenanceFilter noneFilter = (i, r) -> IndexMaintenanceFilter.IndexValues.NONE;
FDBRecordStore fdbRecordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(metaData).setKeySpacePath(path).setIndexMaintenanceFilter(noneFilter).uncheckedOpen();
context.getTimer().reset();
FDBRecordStoreBase.UserVersionChecker userVersionChecker = new FDBRecordStoreBase.UserVersionChecker() {
@Override
public CompletableFuture<Integer> checkUserVersion(int oldUserVersion, int oldMetaDataVersion, RecordMetaDataProvider metaData) {
return CompletableFuture.completedFuture(Integer.valueOf(1));
}
@Override
public IndexState needRebuildIndex(Index index, long recordCount, boolean indexOnNewRecordTypes) {
return IndexState.READABLE;
}
};
fdbRecordStore.checkVersion(userVersionChecker, FDBRecordStoreBase.StoreExistenceCheck.NONE).get();
Collection<StoreTimer.Event> events = context.getTimer().getEvents();
assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
assertTrue(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
}
}
use of com.apple.foundationdb.record.RecordMetaDataProvider in project fdb-record-layer by FoundationDB.
the class IndexingScrubDangling method scrubIndexRangeOnly.
@Nonnull
private CompletableFuture<Boolean> scrubIndexRangeOnly(@Nonnull FDBRecordStore store, byte[] startBytes, byte[] endBytes, @Nonnull AtomicLong recordsScanned) {
// return false when done
Index index = common.getIndex();
final RecordMetaData metaData = store.getRecordMetaData();
final RecordMetaDataProvider recordMetaDataProvider = common.getRecordStoreBuilder().getMetaDataProvider();
if (recordMetaDataProvider == null || !metaData.equals(recordMetaDataProvider.getRecordMetaData())) {
throw new MetaDataException("Store does not have the same metadata");
}
final IndexMaintainer maintainer = store.getIndexMaintainer(index);
// scrubbing only readable, VALUE, idempotence indexes (at least for now)
validateOrThrowEx(maintainer.isIdempotent(), "scrubbed index is not idempotent");
validateOrThrowEx(index.getType().equals(IndexTypes.VALUE) || scrubbingPolicy.ignoreIndexTypeCheck(), "scrubbed index is not a VALUE index");
validateOrThrowEx(store.getIndexState(index) == IndexState.READABLE, "scrubbed index is not readable");
RangeSet rangeSet = new RangeSet(indexScrubIndexRangeSubspace(store, index));
AsyncIterator<Range> ranges = rangeSet.missingRanges(store.ensureContextActive(), startBytes, endBytes).iterator();
final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setIsolationLevel(IsolationLevel.SNAPSHOT).setReturnedRowLimit(// always respectLimit in this path; +1 allows a continuation item
getLimit() + 1);
final ScanProperties scanProperties = new ScanProperties(executeProperties.build());
return ranges.onHasNext().thenCompose(hasNext -> {
if (Boolean.FALSE.equals(hasNext)) {
// Here: no more missing ranges - all done
// To avoid stale metadata, we'll keep the scrubbed-ranges indicator empty until the next scrub call.
Transaction tr = store.getContext().ensureActive();
tr.clear(indexScrubIndexRangeSubspace(store, index).range());
return AsyncUtil.READY_FALSE;
}
final Range range = ranges.next();
final Tuple rangeStart = RangeSet.isFirstKey(range.begin) ? null : Tuple.fromBytes(range.begin);
final Tuple rangeEnd = RangeSet.isFinalKey(range.end) ? null : Tuple.fromBytes(range.end);
final TupleRange tupleRange = TupleRange.between(rangeStart, rangeEnd);
RecordCursor<FDBIndexedRecord<Message>> cursor = store.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, tupleRange, null, IndexOrphanBehavior.RETURN, scanProperties);
final AtomicBoolean hasMore = new AtomicBoolean(true);
final AtomicReference<RecordCursorResult<FDBIndexedRecord<Message>>> lastResult = new AtomicReference<>(RecordCursorResult.exhausted());
final long scanLimit = scrubbingPolicy.getEntriesScanLimit();
return iterateRangeOnly(store, cursor, this::deleteIndexIfDangling, lastResult, hasMore, recordsScanned, true).thenApply(vignore -> hasMore.get() ? lastResult.get().get().getIndexEntry().getKey() : rangeEnd).thenCompose(cont -> rangeSet.insertRange(store.ensureContextActive(), packOrNull(rangeStart), packOrNull(cont), true).thenApply(ignore -> {
if (scanLimit > 0) {
scanCounter += recordsScanned.get();
if (scanLimit <= scanCounter) {
return false;
}
}
return !Objects.equals(cont, rangeEnd);
}));
});
}
Aggregations