use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method assertMaxVersionsForGroups.
@SuppressWarnings("try")
private void assertMaxVersionsForGroups(@Nonnull SortedMap<Integer, FDBRecordVersion> groupsToVersions) {
try (FDBRecordContext context = openContext(maxEverVersionWithGroupingHook)) {
Index index = metaData.getIndex("max_ever_version_with_grouping");
List<IndexEntry> entries = new ArrayList<>(groupsToVersions.size());
for (Map.Entry<Integer, FDBRecordVersion> mapEntry : groupsToVersions.entrySet()) {
entries.add(new IndexEntry(index, Key.Evaluated.scalar(mapEntry.getKey()), Key.Evaluated.scalar(mapEntry.getValue())));
}
assertMaxVersionEntries(index, entries);
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method assertMaxVersionWithFunction.
@SuppressWarnings("try")
private void assertMaxVersionWithFunction(int controlColumn, @Nonnull FDBRecordVersion recordVersion) {
try (FDBRecordContext context = openContext(maxEverVersionWithFunctionHook)) {
Index index = metaData.getIndex("max_ever_version_with_function");
IndexEntry entry = new IndexEntry(index, Key.Evaluated.EMPTY, Key.Evaluated.concatenate(controlColumn, recordVersion));
assertMaxVersionEntries(index, Collections.singletonList(entry));
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method enableRecordVersionsAfterTheFact.
@ParameterizedTest(name = "enableRecordVersionsAfterTheFact [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void enableRecordVersionsAfterTheFact(int testFormatVersion, boolean testSplitLongRecords) throws ExecutionException, InterruptedException {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(871L).setNumValue2(871).build();
MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1415).build();
MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(3415L).setNumValue2(3415).build();
Index globalCountIndex = new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
RecordMetaDataHook origHook = metaDataBuilder -> {
noVersionHook.apply(metaDataBuilder);
metaDataBuilder.addIndex((RecordTypeBuilder) null, globalCountIndex);
};
try (FDBRecordContext context = openContext(origHook)) {
assertFalse(metaData.isStoreRecordVersions());
recordStore.saveRecord(record1);
recordStore.saveRecord(record2);
recordStore.saveRecord(record3, null, FDBRecordStoreBase.VersionstampSaveBehavior.WITH_VERSION);
context.commit();
}
try (FDBRecordContext context = openContext(metaDataBuilder -> {
origHook.apply(metaDataBuilder);
metaDataBuilder.setStoreRecordVersions(true);
functionVersionHook.apply(metaDataBuilder);
})) {
assertTrue(metaData.isStoreRecordVersions());
FDBStoredRecord<Message> storedRecord1 = recordStore.loadRecord(Tuple.from(871L));
assertNotNull(storedRecord1);
assertEquals(record1, storedRecord1.getRecord());
assertFalse(storedRecord1.hasVersion());
FDBStoredRecord<Message> storedRecord2 = recordStore.loadRecord(Tuple.from(1415L));
assertNotNull(storedRecord2);
assertEquals(record2, storedRecord2.getRecord());
assertFalse(storedRecord2.hasVersion());
FDBStoredRecord<Message> storedRecord3 = recordStore.loadRecord(Tuple.from(3415L));
assertNotNull(storedRecord3);
assertEquals(record3, storedRecord3.getRecord());
assertTrue(storedRecord3.hasVersion());
RecordCursor<IndexEntry> cursor = recordStore.scanIndex(metaData.getIndex("MySimpleRecord$maybeVersion"), IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN);
assertEquals(Arrays.asList(Tuple.from(null, 1415L), Tuple.from(FDBRecordVersion.MIN_VERSION.toVersionstamp(), 871L), Tuple.from(storedRecord3.getVersion().toVersionstamp(), 3415L)), cursor.map(IndexEntry::getKey).asList().get());
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method upgradeFormatVersions.
@ParameterizedTest(name = "upgradeFormatVersions [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void upgradeFormatVersions(int testFormatVersion, boolean splitLongRecords) {
formatVersion = testFormatVersion;
final RecordMetaDataHook hook = metaDataBuilder -> {
simpleVersionHook.apply(metaDataBuilder);
metaDataBuilder.setSplitLongRecords(splitLongRecords);
};
final List<Message> records = Arrays.asList(MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1776L).setNumValue2(2).build());
List<FDBStoredRecord<Message>> storedRecords;
try (FDBRecordContext context = openContext(hook)) {
List<FDBStoredRecord<Message>> storedRecordsWithIncompletes = records.stream().map(recordStore::saveRecord).collect(Collectors.toList());
context.commit();
byte[] globalVersion = context.getVersionStamp();
storedRecords = storedRecordsWithIncompletes.stream().map(record -> record.withCommittedVersion(globalVersion)).collect(Collectors.toList());
}
try (FDBRecordContext context = openContext(hook)) {
if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION) {
validateUsingOlderVersionFormat(storedRecords);
} else {
validateUsingNewerVersionFormat(storedRecords);
}
}
// Update to the current format version
formatVersion = FDBRecordStore.MAX_SUPPORTED_FORMAT_VERSION;
if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION)) {
// After format version upgrade, each record should now store that it has the version inlined
storedRecords = storedRecords.stream().map(record -> new FDBStoredRecord<>(record.getPrimaryKey(), record.getRecordType(), record.getRecord(), record.getKeyCount() + 1, record.getKeySize() * 2 + 1, record.getValueSize() + 1 + FDBRecordVersion.VERSION_LENGTH, record.isSplit(), true, record.getVersion())).collect(Collectors.toList());
}
try (FDBRecordContext context = openContext(hook)) {
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(storedRecords);
} else {
validateUsingNewerVersionFormat(storedRecords);
}
for (FDBStoredRecord<Message> storedRecord : storedRecords) {
Optional<FDBRecordVersion> loadedVersion = recordStore.loadRecordVersion(storedRecord.getPrimaryKey());
assertTrue(loadedVersion.isPresent());
assertEquals(storedRecord.getVersion(), loadedVersion.get());
RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().equalsValue(storedRecord.getVersion())).build();
RecordQueryPlan plan = planner.plan(query);
final String endpointString = "[" + storedRecord.getVersion().toVersionstamp(false).toString() + "]";
assertThat(plan, indexScan(allOf(indexName("globalVersion"), bounds(hasTupleString("[" + endpointString + "," + endpointString + "]")))));
List<FDBStoredRecord<Message>> queriedRecords = recordStore.executeQuery(plan).map(FDBQueriedRecord::getStoredRecord).asList().join();
assertEquals(Collections.singletonList(storedRecord), queriedRecords);
}
assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
final List<FDBStoredRecord<Message>> fewerRecords = storedRecords.subList(1, storedRecords.size());
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(fewerRecords);
} else {
validateUsingNewerVersionFormat(fewerRecords);
}
recordStore.saveRecord(storedRecords.get(0).getRecord());
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(fewerRecords);
} else {
validateUsingNewerVersionFormat(fewerRecords);
}
// do not commit (so we can do a second upgrade)
}
final Index newValueIndex = new Index("MySimpleRecord$num2", field("num_value_2"), IndexTypes.VALUE);
final Index newVersionIndex = new Index("MySimpleRecord$version-num2", concat(VersionKeyExpression.VERSION, field("num_value_2")), IndexTypes.VERSION);
final RecordMetaDataHook hookWithNewIndexes = metaDataBuilder -> {
hook.apply(metaDataBuilder);
metaDataBuilder.addIndex("MySimpleRecord", newValueIndex);
metaDataBuilder.addIndex("MySimpleRecord", newVersionIndex);
};
final List<FDBStoredRecord<Message>> newStoredRecords;
try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
assertTrue(recordStore.getRecordStoreState().isReadable(newValueIndex));
boolean performedMigration = testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION);
assertNotEquals(performedMigration, recordStore.getRecordStoreState().isReadable(newVersionIndex));
if (recordStore.getRecordStoreState().isReadable(newVersionIndex)) {
// Validate versions are the same for all records in index and in primary store
List<Pair<Tuple, FDBRecordVersion>> recordScannedValues = recordStore.scanRecords(null, ScanProperties.FORWARD_SCAN).map(record -> Pair.of(record.getPrimaryKey(), record.getVersion())).asList().join();
List<Pair<Tuple, FDBRecordVersion>> indexedScannedValues = recordStore.scanIndex(newVersionIndex, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(indexEntry -> Pair.of(TupleHelpers.subTuple(indexEntry.getKey(), 2, indexEntry.getKey().size()), FDBRecordVersion.fromVersionstamp(indexEntry.getKey().getVersionstamp(0), false))).asList().join();
assertEquals(recordScannedValues, indexedScannedValues);
}
// Save record at newer version
assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
FDBStoredRecord<Message> newRecord0 = recordStore.saveRecord(storedRecords.get(0).getRecord());
FDBStoredRecord<Message> newRecord2 = recordStore.saveRecord(storedRecords.get(2).getRecord());
assertEquals(newRecord0.getVersion(), recordStore.loadRecordVersion(newRecord0.getPrimaryKey()).get());
assertEquals(newRecord2.getVersion(), recordStore.loadRecordVersion(newRecord2.getPrimaryKey()).get());
context.commit();
byte[] versionstamp = context.getVersionStamp();
newStoredRecords = Arrays.asList(newRecord0.withVersion(FDBRecordVersion.complete(versionstamp, 0)), storedRecords.get(1), newRecord2.withVersion(FDBRecordVersion.complete(versionstamp, 1)));
}
try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(newStoredRecords);
} else {
validateUsingNewerVersionFormat(newStoredRecords);
}
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class TextIndexTest method scanWithSkip.
public void scanWithSkip(@Nonnull Index index, @Nonnull String token, int skip, int limit, boolean reverse) throws Exception {
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
final List<IndexEntry> fullResults = scanIndex(recordStore, index, TupleRange.allOf(Tuple.from(token)));
validateSorted(fullResults);
final ScanProperties scanProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(limit).setSkip(skip).build().asScanProperties(reverse);
final RecordCursor<IndexEntry> cursor = recordStore.scanIndex(index, BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(token)), null, scanProperties);
List<IndexEntry> scanResults = cursor.asList().get();
RecordCursorResult<IndexEntry> noNextResult = cursor.getNext();
assertThat(noNextResult.hasNext(), is(false));
assertEquals((limit != ReadTransaction.ROW_LIMIT_UNLIMITED && scanResults.size() == limit) ? RETURN_LIMIT_REACHED : SOURCE_EXHAUSTED, noNextResult.getNoNextReason());
List<IndexEntry> expectedResults;
if (reverse) {
scanResults = new ArrayList<>(scanResults);
Collections.reverse(scanResults);
expectedResults = fullResults.subList((limit == ReadTransaction.ROW_LIMIT_UNLIMITED || limit == Integer.MAX_VALUE) ? 0 : Math.max(0, fullResults.size() - skip - limit), Math.max(0, fullResults.size() - skip));
} else {
expectedResults = fullResults.subList(Math.min(fullResults.size(), skip), (limit == ReadTransaction.ROW_LIMIT_UNLIMITED || limit == Integer.MAX_VALUE) ? fullResults.size() : Math.min(fullResults.size(), skip + limit));
}
assertEquals(expectedResults, scanResults);
}
}
Aggregations