use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion in project fdb-record-layer by FoundationDB.
the class TupleTypeUtilTest method listEquivalence.
@Test
public void listEquivalence() {
List<Object> packableValues = VALUES.stream().filter(value -> {
if (value instanceof FDBRecordVersion) {
return ((FDBRecordVersion) value).isComplete();
} else if (value instanceof Versionstamp) {
return ((Versionstamp) value).isComplete();
} else {
return true;
}
}).collect(Collectors.toList());
List<Object> equivalentList = TupleTypeUtil.toTupleEquivalentList(packableValues);
Tuple tupleWithNormalizing = Tuple.fromList(TupleTypeUtil.toTupleAppropriateList(equivalentList));
Tuple tupleWithoutNormalizing = Tuple.fromList(TupleTypeUtil.toTupleAppropriateList(packableValues));
assertThat(TupleHelpers.equals(tupleWithNormalizing, tupleWithoutNormalizing), is(true));
assertArrayEquals(tupleWithoutNormalizing.pack(), tupleWithNormalizing.pack());
Tuple deserializedTuple = Tuple.fromBytes(tupleWithNormalizing.pack());
assertThat(TupleHelpers.equals(tupleWithNormalizing, deserializedTuple), is(true));
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion in project fdb-record-layer by FoundationDB.
the class FDBFilterCoalescingQueryTest method versionRangeCoalesce.
/**
* Validate that a query for all values within a given range of versions will be coalesced into a single scan.
* This is similar to the {@link #simpleRangeCoalesce()} test above, but it is for version key expressions in
* particular.
*/
@Test
public void versionRangeCoalesce() throws Exception {
Index versionIndex = new Index("MySimpleRecord$version", VersionKeyExpression.VERSION, IndexTypes.VERSION);
RecordMetaDataHook hook = metaData -> {
metaData.setStoreRecordVersions(true);
metaData.addIndex("MySimpleRecord", versionIndex);
};
complexQuerySetup(hook);
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
long readVersion = context.getReadVersion();
FDBRecordVersion lowBoundary = FDBRecordVersion.firstInDBVersion(0);
FDBRecordVersion highBoundary = FDBRecordVersion.lastInDBVersion(readVersion);
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.version().greaterThan(lowBoundary), Query.version().lessThan(highBoundary))).build();
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, indexScan(allOf(indexName(versionIndex.getName()), bounds(hasTupleString("([" + lowBoundary.toVersionstamp() + "],[" + highBoundary.toVersionstamp() + "])")))));
try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
int i = 0;
while (cursor.hasNext()) {
FDBQueriedRecord<Message> rec = cursor.next();
FDBRecordVersion version = rec.getVersion();
assertNotNull(version);
assertThat(version, allOf(lessThan(highBoundary), greaterThan(lowBoundary)));
i++;
}
assertEquals(100, i);
assertDiscardedNone(context);
}
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method readVersionFromStoredRecordInTwoStores.
/**
* Store a record in one store, then store a different record in a different store with the same primary key
* and validate that the version read for the first record matches the version written and not the version
* for the second record (which could happen if the local version cache leaks information between stores).
*/
@ParameterizedTest(name = "readVersionFromStoredRecordInTwoStores [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void readVersionFromStoredRecordInTwoStores(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
final MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).build();
final MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1729).build();
final FDBRecordVersion version1;
try (FDBRecordContext context = openContext(simpleVersionHook)) {
FDBStoredRecord<?> storedRecord1 = recordStore.saveRecord(record1);
assertNotNull(storedRecord1.getVersion());
context.commit();
assertNotNull(context.getVersionStamp());
version1 = storedRecord1.getVersion().withCommittedVersion(context.getVersionStamp());
}
final FDBRecordVersion version2;
try (FDBRecordContext context = openContext(simpleVersionHook)) {
final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).create();
FDBStoredRecord<?> storedRecord2 = recordStore2.saveRecord(record2);
assertNotNull(storedRecord2.getVersion());
FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
assertNotNull(storedRecord1);
assertEquals(version1, storedRecord1.getVersion());
context.commit();
assertNotNull(context.getVersionStamp());
version2 = storedRecord2.getVersion().withCommittedVersion(context.getVersionStamp());
}
try (FDBRecordContext context = openContext(simpleVersionHook)) {
final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).open();
FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
assertNotNull(storedRecord1);
assertEquals(version1, storedRecord1.getVersion());
assertEquals(record1, storedRecord1.getRecord());
FDBStoredRecord<?> storedRecord2 = recordStore2.loadRecord(Tuple.from(record2.getRecNo()));
assertNotNull(storedRecord2);
assertEquals(version2, storedRecord2.getVersion());
assertEquals(record2, storedRecord2.getRecord());
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method assertMaxVersionsForGroups.
@SuppressWarnings("try")
private void assertMaxVersionsForGroups(@Nonnull SortedMap<Integer, FDBRecordVersion> groupsToVersions) {
try (FDBRecordContext context = openContext(maxEverVersionWithGroupingHook)) {
Index index = metaData.getIndex("max_ever_version_with_grouping");
List<IndexEntry> entries = new ArrayList<>(groupsToVersions.size());
for (Map.Entry<Integer, FDBRecordVersion> mapEntry : groupsToVersions.entrySet()) {
entries.add(new IndexEntry(index, Key.Evaluated.scalar(mapEntry.getKey()), Key.Evaluated.scalar(mapEntry.getValue())));
}
assertMaxVersionEntries(index, entries);
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method queryOnVersion.
@ParameterizedTest(name = "queryOnVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnVersion(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
List<TestRecords1Proto.MyOtherRecord> otherRecords = IntStream.range(0, 30).mapToObj(id -> TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(id * 2 + 1).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
Iterator<TestRecords1Proto.MyOtherRecord> otherIterator = otherRecords.iterator();
while (simpleIterator.hasNext()) {
try (FDBRecordContext context = openContext(simpleVersionHook)) {
int done = 0;
while (simpleIterator.hasNext() && done != 5) {
recordStore.saveRecord(simpleIterator.next());
recordStore.saveRecord(otherIterator.next());
done += 1;
}
context.commit();
}
}
// Query all records.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = Stream.concat(simpleRecords.stream().map(MySimpleRecord::getRecNo), otherRecords.stream().map(TestRecords1Proto.MyOtherRecord::getRecNo)).sorted().collect(Collectors.toList());
FDBRecordVersion last = null;
List<Long> receivedKeys = new ArrayList<>();
int totalSeen = 0;
while (true) {
RecordQueryPlan plan;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion <,>)", plan.toString());
} else {
RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().greaterThan(last)).setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion ([" + last.toVersionstamp() + "],>)", plan.toString());
}
RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator();
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBQueriedRecord<Message> record = cursor.next();
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() + otherRecords.size(), totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query MySimpleRecord based on value.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals((simpleRecords.size() + 1) / 2, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that requires also filtering
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0 && rec.getNumValue3Indexed() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() / 6, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that can't be satisfied with an index scan.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
// Preliminary query to get a read version.
RecordQuery prelimQuery = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan prelimPlan = planner.plan(prelimQuery);
FDBRecordVersion chosenVersion = recordStore.executeQuery(prelimPlan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asList().thenApply(list -> list.get(list.size() - 1).getVersion()).join();
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.version().greaterThan(chosenVersion)).setSort(field("num_value_3_indexed")).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num_value_3_indexed <,>) | version GREATER_THAN " + chosenVersion.toString(), plan.toString());
List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
int last = -1;
for (FDBQueriedRecord<Message> record : records) {
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertThat(last, lessThanOrEqualTo(simpleRecord.getNumValue3Indexed()));
assertTrue(record.hasVersion());
assertThat(chosenVersion, lessThan(record.getVersion()));
last = simpleRecord.getNumValue3Indexed();
}
assertEquals(simpleRecords.size() - 5, records.size());
}
}
Aggregations