use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method maxEverVersionWithExtraColumn.
@ParameterizedTest(name = "maxEverVersionWithExtraColumn [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
public void maxEverVersionWithExtraColumn(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
// Save a record with a fresh version
FDBRecordVersion expectedMaxVersion;
try (FDBRecordContext context = openContext(maxEverVersionWithExtraColumnHook)) {
MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(0).build();
recordStore.saveRecord(record1);
context.commit();
assertNotNull(context.getVersionStamp());
expectedMaxVersion = FDBRecordVersion.complete(context.getVersionStamp(), 0);
}
assertMaxVersionWithExtraColumn(0, expectedMaxVersion);
// Should be no change
try (FDBRecordContext context = openContext(maxEverVersionWithExtraColumnHook)) {
MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(-1).build();
recordStore.saveRecord(record2, getBiggerVersion(expectedMaxVersion));
context.commit();
}
assertMaxVersionWithExtraColumn(0, expectedMaxVersion);
// Add a record with a larger value for the first column and a smaller value for the version
// Should update to the new version
FDBRecordVersion smallerVersion;
try (FDBRecordContext context = openContext(maxEverVersionWithExtraColumnHook)) {
MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(1863L).setNumValue2(1).build();
smallerVersion = getSmallerVersion(expectedMaxVersion);
recordStore.saveRecord(record3, smallerVersion);
context.commit();
}
assertMaxVersionWithExtraColumn(1, smallerVersion);
// the update even if that causes the extra column to go down.
try (FDBRecordContext context = openContext(maxEverVersionWithExtraColumnHook)) {
MySimpleRecord record4 = MySimpleRecord.newBuilder().setRecNo(1455L).setNumValue2(0).build();
recordStore.saveRecord(record4);
context.commit();
assertNotNull(context.getVersionStamp());
expectedMaxVersion = FDBRecordVersion.complete(context.getVersionStamp(), 0);
}
assertMaxVersionWithExtraColumn(0, expectedMaxVersion);
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method saveLoadWithRepeatedVersion.
@ParameterizedTest(name = "saveLoadWithRepeatedVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void saveLoadWithRepeatedVersion(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
MySimpleRecord simpleRecord1 = MySimpleRecord.newBuilder().setRecNo(1066L).addRepeater(1).addRepeater(2).addRepeater(3).build();
MySimpleRecord simpleRecord2 = MySimpleRecord.newBuilder().setRecNo(1729L).addRepeater(1).build();
MySimpleRecord simpleRecord3 = MySimpleRecord.newBuilder().setRecNo(1776L).build();
byte[] versionstamp;
try (FDBRecordContext context = openContext(repeatedVersionHook)) {
recordStore.saveRecord(simpleRecord1);
recordStore.saveRecord(simpleRecord2);
recordStore.saveRecord(simpleRecord3);
context.commit();
versionstamp = context.getVersionStamp();
assertEquals(3, context.claimLocalVersion());
}
try (FDBRecordContext context = openContext(repeatedVersionHook)) {
FDBStoredRecord<Message> stored1 = recordStore.loadRecord(Tuple.from(1066L));
assertTrue(stored1.hasVersion());
FDBRecordVersion version1 = stored1.getVersion();
assertNotNull(version1);
assertArrayEquals(versionstamp, version1.getGlobalVersion());
assertEquals(0, version1.getLocalVersion());
FDBStoredRecord<Message> stored2 = recordStore.loadRecord(Tuple.from(1729L));
assertTrue(stored2.hasVersion());
FDBRecordVersion version2 = stored2.getVersion();
assertNotNull(version2);
assertArrayEquals(versionstamp, version2.getGlobalVersion());
assertEquals(1, version2.getLocalVersion());
FDBStoredRecord<Message> stored3 = recordStore.loadRecord(Tuple.from(1776L));
assertTrue(stored3.hasVersion());
FDBRecordVersion version3 = stored3.getVersion();
assertNotNull(version3);
assertArrayEquals(versionstamp, version3.getGlobalVersion());
assertEquals(2, version3.getLocalVersion());
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method maxEverVersionWithinTransaction.
@ParameterizedTest(name = "maxEverVersionWithinTransaction [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
public void maxEverVersionWithinTransaction(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
// Add two records in the same commit with different local versions to ensure the one with the higher one is written
FDBRecordVersion expectedMaxVersion;
try (FDBRecordContext context = openContext(maxEverVersionHook)) {
MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1215L).build();
MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1815L).build();
recordStore.saveRecord(record1, FDBRecordVersion.incomplete(42));
recordStore.saveRecord(record2, FDBRecordVersion.incomplete(13));
context.commit();
assertNotNull(context.getVersionStamp());
expectedMaxVersion = FDBRecordVersion.complete(context.getVersionStamp(), 42);
}
assertMaxVersion(expectedMaxVersion);
// but the incomplete one is chosen anyway.
try (FDBRecordContext context = openContext(maxEverVersionHook)) {
MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(1066L).build();
MySimpleRecord record4 = MySimpleRecord.newBuilder().setRecNo(1415L).build();
FDBStoredRecord<?> storedRecord3 = recordStore.saveRecord(record3);
assertNotNull(storedRecord3.getVersion());
FDBRecordVersion version4 = getBiggerVersion(expectedMaxVersion);
FDBStoredRecord<?> storedRecord4 = recordStore.saveRecord(record4, version4);
context.commit();
assertNotNull(context.getVersionStamp());
FDBRecordVersion version3 = storedRecord3.getVersion().withCommittedVersion(context.getVersionStamp());
Assumptions.assumeTrue(version3.compareTo(version4) < 0, "committed version should be less than incremented version");
expectedMaxVersion = version3;
}
assertMaxVersion(expectedMaxVersion);
// Same as above, but write the record that should have the maximum version first as it shouldn't matter.
try (FDBRecordContext context = openContext(maxEverVersionHook)) {
MySimpleRecord record5 = MySimpleRecord.newBuilder().setRecNo(1564L).build();
MySimpleRecord record6 = MySimpleRecord.newBuilder().setRecNo(1455L).build();
FDBRecordVersion version5 = getBiggerVersion(expectedMaxVersion);
recordStore.saveRecord(record5, version5);
FDBStoredRecord<?> storedRecord6 = recordStore.saveRecord(record6);
assertNotNull(storedRecord6.getVersion());
context.commit();
assertNotNull(context.getVersionStamp());
FDBRecordVersion version6 = storedRecord6.getVersion().withCommittedVersion(context.getVersionStamp());
Assumptions.assumeTrue(version6.compareTo(version5) < 0, "committed version should be less than incremented version");
expectedMaxVersion = version6;
}
assertMaxVersion(expectedMaxVersion);
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method withMetaDataRebuilds.
@ParameterizedTest(name = "withMetaDataRebuilds [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void withMetaDataRebuilds(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
RecordMetaDataHook firstHook = metaDataBuilder -> {
metaDataBuilder.setSplitLongRecords(splitLongRecords);
metaDataBuilder.addUniversalIndex(new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
metaDataBuilder.addUniversalIndex(new Index("globalVersion", VersionKeyExpression.VERSION, IndexTypes.VERSION));
};
RecordMetaDataHook secondHook = metaDataBuilder -> {
firstHook.apply(metaDataBuilder);
metaDataBuilder.removeIndex("globalVersion");
metaDataBuilder.setStoreRecordVersions(false);
};
RecordMetaDataHook thirdHook = metaDataBuilder -> {
secondHook.apply(metaDataBuilder);
metaDataBuilder.setStoreRecordVersions(true);
metaDataBuilder.addUniversalIndex(new Index("globalVersion2", VersionKeyExpression.VERSION, IndexTypes.VERSION));
};
MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).build();
FDBRecordVersion version1;
MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1776L).build();
MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(1955L).build();
FDBRecordVersion version3;
RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
// First with version on.
try (FDBRecordContext context = openContext(firstHook)) {
FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record1);
assertTrue(storedRecord.hasVersion());
context.commit();
version1 = FDBRecordVersion.complete(context.getVersionStamp(), storedRecord.getVersion().getLocalVersion());
}
try (FDBRecordContext context = openContext(firstHook)) {
FDBStoredRecord<?> loadedRecord = recordStore.loadRecord(Tuple.from(1066L));
assertNotNull(loadedRecord);
assertTrue(loadedRecord.hasVersion());
assertEquals(version1, loadedRecord.getVersion());
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, indexScan("globalVersion"));
List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
assertEquals(1, records.size());
FDBQueriedRecord<Message> queriedRecord = records.get(0);
assertEquals(Tuple.from(1066L), queriedRecord.getPrimaryKey());
assertTrue(queriedRecord.hasVersion());
assertEquals(version1, queriedRecord.getVersion());
}
// Now with version off.
try (FDBRecordContext context = openContext(secondHook)) {
FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record2);
assertFalse(storedRecord.hasVersion());
context.commit();
}
try (FDBRecordContext context = openContext(secondHook)) {
FDBStoredRecord<?> loadedRecord1 = recordStore.loadRecord(Tuple.from(1066L));
assertNotNull(loadedRecord1);
assertEquals(testFormatVersion >= FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION, loadedRecord1.hasVersion());
FDBStoredRecord<?> loadedRecord2 = recordStore.loadRecord(Tuple.from(1776L));
assertNotNull(loadedRecord2);
assertFalse(loadedRecord2.hasVersion());
assertThrows(RecordCoreException.class, () -> {
RecordQueryPlan plan = planner.plan(query);
fail("Came up with plan " + plan.toString() + " when it should be impossible");
});
}
// Now with version back on.
try (FDBRecordContext context = openContext(thirdHook)) {
FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record3);
assertTrue(storedRecord.hasVersion());
context.commit();
version3 = FDBRecordVersion.complete(context.getVersionStamp(), storedRecord.getVersion().getLocalVersion());
}
try (FDBRecordContext context = openContext(thirdHook)) {
FDBStoredRecord<?> loadedRecord1 = recordStore.loadRecord(Tuple.from(1066L));
assertEquals(testFormatVersion >= FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION, loadedRecord1.hasVersion());
FDBStoredRecord<?> loadedRecord2 = recordStore.loadRecord(Tuple.from(1776L));
assertFalse(loadedRecord2.hasVersion());
FDBStoredRecord<?> loadedRecord3 = recordStore.loadRecord(Tuple.from(1955L));
assertTrue(loadedRecord3.hasVersion());
assertEquals(version3, loadedRecord3.getVersion());
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, indexScan("globalVersion2"));
List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
assertEquals(3, records.size());
if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION) {
FDBQueriedRecord<Message> queriedRecord1 = records.get(0);
assertEquals(Tuple.from(1066L), queriedRecord1.getPrimaryKey());
assertFalse(queriedRecord1.hasVersion());
FDBQueriedRecord<Message> queriedRecord2 = records.get(1);
assertEquals(Tuple.from(1776L), queriedRecord2.getPrimaryKey());
assertFalse(queriedRecord2.hasVersion());
} else {
FDBQueriedRecord<Message> queriedRecord1 = records.get(0);
assertEquals(Tuple.from(1776L), queriedRecord1.getPrimaryKey());
assertFalse(queriedRecord1.hasVersion());
FDBQueriedRecord<Message> queriedRecord2 = records.get(1);
assertEquals(Tuple.from(1066L), queriedRecord2.getPrimaryKey());
assertTrue(queriedRecord2.hasVersion());
assertEquals(version1, queriedRecord2.getVersion());
}
FDBQueriedRecord<Message> queriedRecord3 = records.get(2);
assertEquals(Tuple.from(1955L), queriedRecord3.getPrimaryKey());
assertTrue(queriedRecord3.hasVersion());
assertEquals(version3, queriedRecord3.getVersion());
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method upgradeFormatVersions.
@ParameterizedTest(name = "upgradeFormatVersions [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void upgradeFormatVersions(int testFormatVersion, boolean splitLongRecords) {
formatVersion = testFormatVersion;
final RecordMetaDataHook hook = metaDataBuilder -> {
simpleVersionHook.apply(metaDataBuilder);
metaDataBuilder.setSplitLongRecords(splitLongRecords);
};
final List<Message> records = Arrays.asList(MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1776L).setNumValue2(2).build());
List<FDBStoredRecord<Message>> storedRecords;
try (FDBRecordContext context = openContext(hook)) {
List<FDBStoredRecord<Message>> storedRecordsWithIncompletes = records.stream().map(recordStore::saveRecord).collect(Collectors.toList());
context.commit();
byte[] globalVersion = context.getVersionStamp();
storedRecords = storedRecordsWithIncompletes.stream().map(record -> record.withCommittedVersion(globalVersion)).collect(Collectors.toList());
}
try (FDBRecordContext context = openContext(hook)) {
if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION) {
validateUsingOlderVersionFormat(storedRecords);
} else {
validateUsingNewerVersionFormat(storedRecords);
}
}
// Update to the current format version
formatVersion = FDBRecordStore.MAX_SUPPORTED_FORMAT_VERSION;
if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION)) {
// After format version upgrade, each record should now store that it has the version inlined
storedRecords = storedRecords.stream().map(record -> new FDBStoredRecord<>(record.getPrimaryKey(), record.getRecordType(), record.getRecord(), record.getKeyCount() + 1, record.getKeySize() * 2 + 1, record.getValueSize() + 1 + FDBRecordVersion.VERSION_LENGTH, record.isSplit(), true, record.getVersion())).collect(Collectors.toList());
}
try (FDBRecordContext context = openContext(hook)) {
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(storedRecords);
} else {
validateUsingNewerVersionFormat(storedRecords);
}
for (FDBStoredRecord<Message> storedRecord : storedRecords) {
Optional<FDBRecordVersion> loadedVersion = recordStore.loadRecordVersion(storedRecord.getPrimaryKey());
assertTrue(loadedVersion.isPresent());
assertEquals(storedRecord.getVersion(), loadedVersion.get());
RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().equalsValue(storedRecord.getVersion())).build();
RecordQueryPlan plan = planner.plan(query);
final String endpointString = "[" + storedRecord.getVersion().toVersionstamp(false).toString() + "]";
assertThat(plan, indexScan(allOf(indexName("globalVersion"), bounds(hasTupleString("[" + endpointString + "," + endpointString + "]")))));
List<FDBStoredRecord<Message>> queriedRecords = recordStore.executeQuery(plan).map(FDBQueriedRecord::getStoredRecord).asList().join();
assertEquals(Collections.singletonList(storedRecord), queriedRecords);
}
assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
final List<FDBStoredRecord<Message>> fewerRecords = storedRecords.subList(1, storedRecords.size());
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(fewerRecords);
} else {
validateUsingNewerVersionFormat(fewerRecords);
}
recordStore.saveRecord(storedRecords.get(0).getRecord());
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(fewerRecords);
} else {
validateUsingNewerVersionFormat(fewerRecords);
}
// do not commit (so we can do a second upgrade)
}
final Index newValueIndex = new Index("MySimpleRecord$num2", field("num_value_2"), IndexTypes.VALUE);
final Index newVersionIndex = new Index("MySimpleRecord$version-num2", concat(VersionKeyExpression.VERSION, field("num_value_2")), IndexTypes.VERSION);
final RecordMetaDataHook hookWithNewIndexes = metaDataBuilder -> {
hook.apply(metaDataBuilder);
metaDataBuilder.addIndex("MySimpleRecord", newValueIndex);
metaDataBuilder.addIndex("MySimpleRecord", newVersionIndex);
};
final List<FDBStoredRecord<Message>> newStoredRecords;
try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
assertTrue(recordStore.getRecordStoreState().isReadable(newValueIndex));
boolean performedMigration = testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION);
assertNotEquals(performedMigration, recordStore.getRecordStoreState().isReadable(newVersionIndex));
if (recordStore.getRecordStoreState().isReadable(newVersionIndex)) {
// Validate versions are the same for all records in index and in primary store
List<Pair<Tuple, FDBRecordVersion>> recordScannedValues = recordStore.scanRecords(null, ScanProperties.FORWARD_SCAN).map(record -> Pair.of(record.getPrimaryKey(), record.getVersion())).asList().join();
List<Pair<Tuple, FDBRecordVersion>> indexedScannedValues = recordStore.scanIndex(newVersionIndex, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(indexEntry -> Pair.of(TupleHelpers.subTuple(indexEntry.getKey(), 2, indexEntry.getKey().size()), FDBRecordVersion.fromVersionstamp(indexEntry.getKey().getVersionstamp(0), false))).asList().join();
assertEquals(recordScannedValues, indexedScannedValues);
}
// Save record at newer version
assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
FDBStoredRecord<Message> newRecord0 = recordStore.saveRecord(storedRecords.get(0).getRecord());
FDBStoredRecord<Message> newRecord2 = recordStore.saveRecord(storedRecords.get(2).getRecord());
assertEquals(newRecord0.getVersion(), recordStore.loadRecordVersion(newRecord0.getPrimaryKey()).get());
assertEquals(newRecord2.getVersion(), recordStore.loadRecordVersion(newRecord2.getPrimaryKey()).get());
context.commit();
byte[] versionstamp = context.getVersionStamp();
newStoredRecords = Arrays.asList(newRecord0.withVersion(FDBRecordVersion.complete(versionstamp, 0)), storedRecords.get(1), newRecord2.withVersion(FDBRecordVersion.complete(versionstamp, 1)));
}
try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
validateUsingOlderVersionFormat(newStoredRecords);
} else {
validateUsingNewerVersionFormat(newStoredRecords);
}
}
}
Aggregations