use of com.apple.foundationdb.record.TestRecords1Proto.MySimpleRecord in project fdb-record-layer by FoundationDB.
the class TransformedRecordSerializerTest method readOldFormat.
@Test
public void readOldFormat() {
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().setRecNo(1066L).setStrValueIndexed("Hello").build();
RecordTypeUnion unionRecord = RecordTypeUnion.newBuilder().setMySimpleRecord(simpleRecord).build();
byte[] serialized = serialize(DynamicMessageRecordSerializer.instance(), simpleRecord);
assertArrayEquals(unionRecord.toByteArray(), serialized);
// Without anything specified
TransformedRecordSerializer<Message> serializer = TransformedRecordSerializer.newDefaultBuilder().build();
Message deserialized = deserialize(serializer, Tuple.from(1066L), serialized);
assertEquals(simpleRecord, deserialized);
// With compression
serializer = TransformedRecordSerializer.newDefaultBuilder().setCompressWhenSerializing(true).build();
deserialized = deserialize(serializer, Tuple.from(1066L), serialized);
assertEquals(simpleRecord, deserialized);
}
use of com.apple.foundationdb.record.TestRecords1Proto.MySimpleRecord in project fdb-record-layer by FoundationDB.
the class TransformedRecordSerializerTest method compressWhenSerializing.
@Test
public void compressWhenSerializing() {
TransformedRecordSerializer<Message> serializer = TransformedRecordSerializer.newDefaultBuilder().setCompressWhenSerializing(true).setCompressionLevel(Deflater.HUFFMAN_ONLY).build();
// There should be no compression actually added for a small record like this
MySimpleRecord smallRecord = MySimpleRecord.newBuilder().setRecNo(1066L).build();
RecordTypeUnion smallUnionRecord = RecordTypeUnion.newBuilder().setMySimpleRecord(smallRecord).build();
byte[] serialized = serialize(serializer, smallRecord);
assertEquals(TransformedRecordSerializer.ENCODING_CLEAR, serialized[0]);
assertArrayEquals(smallUnionRecord.toByteArray(), Arrays.copyOfRange(serialized, 1, serialized.length));
Message deserialized = deserialize(serializer, Tuple.from(1066L), serialized);
assertEquals(smallRecord, deserialized);
assertEquals(storeTimer.getCount(RecordSerializer.Counts.ESCHEW_RECORD_COMPRESSION), 1);
assertEquals(storeTimer.getCount(RecordSerializer.Counts.RECORD_BYTES_BEFORE_COMPRESSION), storeTimer.getCount(RecordSerializer.Counts.RECORD_BYTES_AFTER_COMPRESSION));
logMetrics("metrics with failed compression");
// There should definitely be compression from a record like this
MySimpleRecord largeRecord = MySimpleRecord.newBuilder().setRecNo(1066L).setStrValueIndexed(Strings.repeat("foo", 1000)).build();
final RecordTypeUnion largeUnionRecord = RecordTypeUnion.newBuilder().setMySimpleRecord(largeRecord).build();
serialized = serialize(serializer, largeRecord);
assertThat(storeTimer.getCount(RecordSerializer.Counts.RECORD_BYTES_BEFORE_COMPRESSION), greaterThan(storeTimer.getCount(RecordSerializer.Counts.RECORD_BYTES_AFTER_COMPRESSION)));
assertEquals(TransformedRecordSerializer.ENCODING_COMPRESSED, serialized[0]);
int rawLength = largeUnionRecord.toByteArray().length;
assertEquals(rawLength, ByteBuffer.wrap(serialized, 2, 4).order(ByteOrder.BIG_ENDIAN).getInt());
deserialized = deserialize(serializer, Tuple.from(1066L), serialized);
assertEquals(largeRecord, deserialized);
logMetrics("metrics with successful compression (repeated foo)", "raw_length", rawLength, "compressed_length", serialized.length);
// There should be a medium amount of compression from this record
MySimpleRecord mediumRecord = MySimpleRecord.newBuilder().setRecNo(1066L).setStrValueIndexed(SONNET_108).build();
RecordTypeUnion mediumUnionRecord = RecordTypeUnion.newBuilder().setMySimpleRecord(mediumRecord).build();
serialized = serialize(serializer, mediumRecord);
assertEquals(TransformedRecordSerializer.ENCODING_COMPRESSED, serialized[0]);
rawLength = mediumUnionRecord.toByteArray().length;
assertEquals(rawLength, ByteBuffer.wrap(serialized, 2, 4).order(ByteOrder.BIG_ENDIAN).getInt());
deserialized = deserialize(serializer, Tuple.from(1066L), serialized);
assertEquals(mediumRecord, deserialized);
logMetrics("metrics with successful compression (sonnet 108)", "raw_length", rawLength, "compressed_length", serialized.length);
}
use of com.apple.foundationdb.record.TestRecords1Proto.MySimpleRecord in project fdb-record-layer by FoundationDB.
the class TransformedRecordSerializerTest method unknownCompressionVersion.
@Test
public void unknownCompressionVersion() {
RecordSerializationException e = assertThrows(RecordSerializationException.class, () -> {
TransformedRecordSerializer<Message> serializer = TransformedRecordSerializer.newDefaultBuilder().setCompressWhenSerializing(true).build();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().setRecNo(1066L).setStrValueIndexed(SONNET_108).build();
byte[] serialized = serialize(serializer, simpleRecord);
// Bump the compression version to an unknown value.
serialized[1] += 1;
deserialize(serializer, Tuple.from(1066L), serialized);
});
assertThat(e.getMessage(), containsString("unknown compression version"));
}
use of com.apple.foundationdb.record.TestRecords1Proto.MySimpleRecord in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method readVersionFromStoredRecordInTwoStores.
/**
* Store a record in one store, then store a different record in a different store with the same primary key
* and validate that the version read for the first record matches the version written and not the version
* for the second record (which could happen if the local version cache leaks information between stores).
*/
@ParameterizedTest(name = "readVersionFromStoredRecordInTwoStores [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void readVersionFromStoredRecordInTwoStores(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
final MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).build();
final MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1729).build();
final FDBRecordVersion version1;
try (FDBRecordContext context = openContext(simpleVersionHook)) {
FDBStoredRecord<?> storedRecord1 = recordStore.saveRecord(record1);
assertNotNull(storedRecord1.getVersion());
context.commit();
assertNotNull(context.getVersionStamp());
version1 = storedRecord1.getVersion().withCommittedVersion(context.getVersionStamp());
}
final FDBRecordVersion version2;
try (FDBRecordContext context = openContext(simpleVersionHook)) {
final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).create();
FDBStoredRecord<?> storedRecord2 = recordStore2.saveRecord(record2);
assertNotNull(storedRecord2.getVersion());
FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
assertNotNull(storedRecord1);
assertEquals(version1, storedRecord1.getVersion());
context.commit();
assertNotNull(context.getVersionStamp());
version2 = storedRecord2.getVersion().withCommittedVersion(context.getVersionStamp());
}
try (FDBRecordContext context = openContext(simpleVersionHook)) {
final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).open();
FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
assertNotNull(storedRecord1);
assertEquals(version1, storedRecord1.getVersion());
assertEquals(record1, storedRecord1.getRecord());
FDBStoredRecord<?> storedRecord2 = recordStore2.loadRecord(Tuple.from(record2.getRecNo()));
assertNotNull(storedRecord2);
assertEquals(version2, storedRecord2.getVersion());
assertEquals(record2, storedRecord2.getRecord());
}
}
use of com.apple.foundationdb.record.TestRecords1Proto.MySimpleRecord in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method queryOnVersion.
@ParameterizedTest(name = "queryOnVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnVersion(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
List<TestRecords1Proto.MyOtherRecord> otherRecords = IntStream.range(0, 30).mapToObj(id -> TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(id * 2 + 1).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
Iterator<TestRecords1Proto.MyOtherRecord> otherIterator = otherRecords.iterator();
while (simpleIterator.hasNext()) {
try (FDBRecordContext context = openContext(simpleVersionHook)) {
int done = 0;
while (simpleIterator.hasNext() && done != 5) {
recordStore.saveRecord(simpleIterator.next());
recordStore.saveRecord(otherIterator.next());
done += 1;
}
context.commit();
}
}
// Query all records.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = Stream.concat(simpleRecords.stream().map(MySimpleRecord::getRecNo), otherRecords.stream().map(TestRecords1Proto.MyOtherRecord::getRecNo)).sorted().collect(Collectors.toList());
FDBRecordVersion last = null;
List<Long> receivedKeys = new ArrayList<>();
int totalSeen = 0;
while (true) {
RecordQueryPlan plan;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion <,>)", plan.toString());
} else {
RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().greaterThan(last)).setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion ([" + last.toVersionstamp() + "],>)", plan.toString());
}
RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator();
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBQueriedRecord<Message> record = cursor.next();
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() + otherRecords.size(), totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query MySimpleRecord based on value.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals((simpleRecords.size() + 1) / 2, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that requires also filtering
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0 && rec.getNumValue3Indexed() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() / 6, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that can't be satisfied with an index scan.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
// Preliminary query to get a read version.
RecordQuery prelimQuery = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan prelimPlan = planner.plan(prelimQuery);
FDBRecordVersion chosenVersion = recordStore.executeQuery(prelimPlan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asList().thenApply(list -> list.get(list.size() - 1).getVersion()).join();
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.version().greaterThan(chosenVersion)).setSort(field("num_value_3_indexed")).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num_value_3_indexed <,>) | version GREATER_THAN " + chosenVersion.toString(), plan.toString());
List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
int last = -1;
for (FDBQueriedRecord<Message> record : records) {
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertThat(last, lessThanOrEqualTo(simpleRecord.getNumValue3Indexed()));
assertTrue(record.hasVersion());
assertThat(chosenVersion, lessThan(record.getVersion()));
last = simpleRecord.getNumValue3Indexed();
}
assertEquals(simpleRecords.size() - 5, records.size());
}
}
Aggregations