use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method queryOnVersion.
@ParameterizedTest(name = "queryOnVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnVersion(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
List<TestRecords1Proto.MyOtherRecord> otherRecords = IntStream.range(0, 30).mapToObj(id -> TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(id * 2 + 1).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
Iterator<TestRecords1Proto.MyOtherRecord> otherIterator = otherRecords.iterator();
while (simpleIterator.hasNext()) {
try (FDBRecordContext context = openContext(simpleVersionHook)) {
int done = 0;
while (simpleIterator.hasNext() && done != 5) {
recordStore.saveRecord(simpleIterator.next());
recordStore.saveRecord(otherIterator.next());
done += 1;
}
context.commit();
}
}
// Query all records.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = Stream.concat(simpleRecords.stream().map(MySimpleRecord::getRecNo), otherRecords.stream().map(TestRecords1Proto.MyOtherRecord::getRecNo)).sorted().collect(Collectors.toList());
FDBRecordVersion last = null;
List<Long> receivedKeys = new ArrayList<>();
int totalSeen = 0;
while (true) {
RecordQueryPlan plan;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion <,>)", plan.toString());
} else {
RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().greaterThan(last)).setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion ([" + last.toVersionstamp() + "],>)", plan.toString());
}
RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator();
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBQueriedRecord<Message> record = cursor.next();
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() + otherRecords.size(), totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query MySimpleRecord based on value.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals((simpleRecords.size() + 1) / 2, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that requires also filtering
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0 && rec.getNumValue3Indexed() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() / 6, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that can't be satisfied with an index scan.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
// Preliminary query to get a read version.
RecordQuery prelimQuery = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan prelimPlan = planner.plan(prelimQuery);
FDBRecordVersion chosenVersion = recordStore.executeQuery(prelimPlan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asList().thenApply(list -> list.get(list.size() - 1).getVersion()).join();
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.version().greaterThan(chosenVersion)).setSort(field("num_value_3_indexed")).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num_value_3_indexed <,>) | version GREATER_THAN " + chosenVersion.toString(), plan.toString());
List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
int last = -1;
for (FDBQueriedRecord<Message> record : records) {
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertThat(last, lessThanOrEqualTo(simpleRecord.getNumValue3Indexed()));
assertTrue(record.hasVersion());
assertThat(chosenVersion, lessThan(record.getVersion()));
last = simpleRecord.getNumValue3Indexed();
}
assertEquals(simpleRecords.size() - 5, records.size());
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method removeWithVersion.
@ParameterizedTest(name = "removeWithVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
public void removeWithVersion(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
MySimpleRecord record = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).build();
FDBStoredRecord<Message> storedRecord;
byte[] versionstamp;
try (FDBRecordContext context = openContext(simpleVersionHook)) {
storedRecord = recordStore.saveRecord(record);
context.commit();
versionstamp = context.getVersionStamp();
assertEquals(1, context.claimLocalVersion());
}
try (FDBRecordContext context = openContext(simpleVersionHook)) {
RecordFunction<FDBRecordVersion> function = Query.version().getFunction();
FDBRecordVersion version = recordStore.evaluateRecordFunction(function, storedRecord).join();
assertNotNull(version);
assertArrayEquals(versionstamp, version.getGlobalVersion());
assertEquals(0, version.getLocalVersion());
Optional<FDBRecordVersion> versionOptional = recordStore.loadRecordVersion(storedRecord.getPrimaryKey());
assertTrue(versionOptional.isPresent());
assertEquals(version, versionOptional.get());
// Remove record saved within previous transaction
assertTrue(recordStore.deleteRecord(Tuple.from(1066L)));
version = recordStore.evaluateRecordFunction(function, storedRecord).join();
assertNull(version);
versionOptional = recordStore.loadRecordVersion(Tuple.from(1066L));
assertFalse(versionOptional.isPresent());
assertFalse(recordStore.deleteRecord(Tuple.from(1066L)));
version = recordStore.evaluateRecordFunction(function, storedRecord).join();
assertNull(version);
versionOptional = recordStore.loadRecordVersion(Tuple.from(1066L));
assertFalse(versionOptional.isPresent());
// Save a new record and verify version removed with it after it is deleted
MySimpleRecord record2 = record.toBuilder().setRecNo(1415L).build();
FDBStoredRecord<Message> storedRecord2 = recordStore.saveRecord(record2);
assertTrue(storedRecord2.hasVersion());
assertFalse(storedRecord2.getVersion().isComplete());
version = recordStore.evaluateRecordFunction(function, storedRecord2).join();
assertNotNull(version);
assertEquals(storedRecord2.getVersion(), version);
versionOptional = recordStore.loadRecordVersion(Tuple.from(1415L));
assertTrue(versionOptional.isPresent());
assertEquals(version, versionOptional.get());
assertTrue(recordStore.deleteRecord(Tuple.from(1415L)));
version = recordStore.evaluateRecordFunction(function, storedRecord2).join();
assertNull(version);
versionOptional = recordStore.loadRecordVersion(Tuple.from(1415L));
assertFalse(versionOptional.isPresent());
context.commit();
}
try (FDBRecordContext context = openContext(simpleVersionHook)) {
// Verify that the version added in the second record wasn't actually committed during the
// pre-commit hook that writes all the versioned keys and values
Optional<FDBRecordVersion> versionOptional = recordStore.loadRecordVersion(Tuple.from(1415L));
assertFalse(versionOptional.isPresent());
context.commit();
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method saveLoadWithRepeatedAndCompoundVersion.
@ParameterizedTest(name = "saveLoadWithRepeatedAndCompoundVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void saveLoadWithRepeatedAndCompoundVersion(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
MySimpleRecord simpleRecord1 = MySimpleRecord.newBuilder().setRecNo(1066L).addRepeater(1).addRepeater(2).addRepeater(3).build();
MySimpleRecord simpleRecord2 = MySimpleRecord.newBuilder().setRecNo(1729L).addRepeater(1).build();
MySimpleRecord simpleRecord3 = MySimpleRecord.newBuilder().setRecNo(1776L).build();
byte[] versionstamp;
try (FDBRecordContext context = openContext(repeatedAndCompoundVersionHook)) {
recordStore.saveRecord(simpleRecord1);
recordStore.saveRecord(simpleRecord2);
recordStore.saveRecord(simpleRecord3);
context.commit();
versionstamp = context.getVersionStamp();
assertEquals(3, context.claimLocalVersion());
}
try (FDBRecordContext context = openContext(repeatedAndCompoundVersionHook)) {
FDBStoredRecord<Message> stored1 = recordStore.loadRecord(Tuple.from(1066L));
assertTrue(stored1.hasVersion());
FDBRecordVersion version1 = stored1.getVersion();
assertNotNull(version1);
assertArrayEquals(versionstamp, version1.getGlobalVersion());
assertEquals(0, version1.getLocalVersion());
FDBStoredRecord<Message> stored2 = recordStore.loadRecord(Tuple.from(1729L));
assertTrue(stored2.hasVersion());
FDBRecordVersion version2 = stored2.getVersion();
assertNotNull(version2);
assertArrayEquals(versionstamp, version2.getGlobalVersion());
assertEquals(1, version2.getLocalVersion());
FDBStoredRecord<Message> stored3 = recordStore.loadRecord(Tuple.from(1776L));
assertTrue(stored3.hasVersion());
FDBRecordVersion version3 = stored3.getVersion();
assertNotNull(version3);
assertArrayEquals(versionstamp, version3.getGlobalVersion());
assertEquals(2, version3.getLocalVersion());
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method assertMaxVersionWithFunction.
@SuppressWarnings("try")
private void assertMaxVersionWithFunction(int controlColumn, @Nonnull FDBRecordVersion recordVersion) {
try (FDBRecordContext context = openContext(maxEverVersionWithFunctionHook)) {
Index index = metaData.getIndex("max_ever_version_with_function");
IndexEntry entry = new IndexEntry(index, Key.Evaluated.EMPTY, Key.Evaluated.concatenate(controlColumn, recordVersion));
assertMaxVersionEntries(index, Collections.singletonList(entry));
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method enableRecordVersionsAfterTheFact.
@ParameterizedTest(name = "enableRecordVersionsAfterTheFact [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void enableRecordVersionsAfterTheFact(int testFormatVersion, boolean testSplitLongRecords) throws ExecutionException, InterruptedException {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(871L).setNumValue2(871).build();
MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1415).build();
MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(3415L).setNumValue2(3415).build();
Index globalCountIndex = new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
RecordMetaDataHook origHook = metaDataBuilder -> {
noVersionHook.apply(metaDataBuilder);
metaDataBuilder.addIndex((RecordTypeBuilder) null, globalCountIndex);
};
try (FDBRecordContext context = openContext(origHook)) {
assertFalse(metaData.isStoreRecordVersions());
recordStore.saveRecord(record1);
recordStore.saveRecord(record2);
recordStore.saveRecord(record3, null, FDBRecordStoreBase.VersionstampSaveBehavior.WITH_VERSION);
context.commit();
}
try (FDBRecordContext context = openContext(metaDataBuilder -> {
origHook.apply(metaDataBuilder);
metaDataBuilder.setStoreRecordVersions(true);
functionVersionHook.apply(metaDataBuilder);
})) {
assertTrue(metaData.isStoreRecordVersions());
FDBStoredRecord<Message> storedRecord1 = recordStore.loadRecord(Tuple.from(871L));
assertNotNull(storedRecord1);
assertEquals(record1, storedRecord1.getRecord());
assertFalse(storedRecord1.hasVersion());
FDBStoredRecord<Message> storedRecord2 = recordStore.loadRecord(Tuple.from(1415L));
assertNotNull(storedRecord2);
assertEquals(record2, storedRecord2.getRecord());
assertFalse(storedRecord2.hasVersion());
FDBStoredRecord<Message> storedRecord3 = recordStore.loadRecord(Tuple.from(3415L));
assertNotNull(storedRecord3);
assertEquals(record3, storedRecord3.getRecord());
assertTrue(storedRecord3.hasVersion());
RecordCursor<IndexEntry> cursor = recordStore.scanIndex(metaData.getIndex("MySimpleRecord$maybeVersion"), IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN);
assertEquals(Arrays.asList(Tuple.from(null, 1415L), Tuple.from(FDBRecordVersion.MIN_VERSION.toVersionstamp(), 871L), Tuple.from(storedRecord3.getVersion().toVersionstamp(), 3415L)), cursor.map(IndexEntry::getKey).asList().get());
}
}
Aggregations