use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class KeySpaceDirectoryTest method testListAcrossTransactions.
@Test
public void testListAcrossTransactions() throws Exception {
KeySpace root = new KeySpace(new KeySpaceDirectory("a", KeyType.LONG, random.nextLong()).addSubdirectory(new KeySpaceDirectory("b", KeyType.STRING)));
final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
final List<String> directoryEntries = IntStream.range(0, 10).boxed().map(i -> "val_" + i).collect(Collectors.toList());
final KeySpacePath rootPath = root.path("a");
try (final FDBRecordContext context = database.openContext()) {
final Transaction tr = context.ensureActive();
directoryEntries.forEach(name -> tr.set(rootPath.add("b", name).toTuple(context).pack(), TupleHelpers.EMPTY.pack()));
context.commit();
}
byte[] continuation = null;
int idx = 0;
do {
try (final FDBRecordContext context = database.openContext()) {
final RecordCursor<ResolvedKeySpacePath> cursor = rootPath.listSubdirectoryAsync(context, "b", continuation, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()));
List<ResolvedKeySpacePath> subdirs = context.asyncToSync(FDBStoreTimer.Waits.WAIT_KEYSPACE_LIST, cursor.asList());
if (!subdirs.isEmpty()) {
assertEquals(2, subdirs.size(), "Wrong number of path entries returned");
assertEquals("val_" + idx, subdirs.get(0).getResolvedValue());
assertEquals("val_" + (idx + 1), subdirs.get(1).getResolvedValue());
idx += 2;
continuation = cursor.getNext().getContinuation().toBytes();
System.out.println(continuation == null ? "null" : Tuple.fromBytes(continuation));
} else {
continuation = cursor.getNext().getContinuation().toBytes();
assertNull(continuation);
}
}
} while (continuation != null);
assertEquals(directoryEntries.size(), idx);
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class FDBNestedFieldQueryTest method testNestedPrimaryKeyQuery.
/**
* Verify that record scans with nested primary keys works properly.
* Specifically, verify that a filter is implemented as a record scan in the case where there is a two-field
* primary key both of whose fields are nested in some header subrecord.
*/
@DualPlannerTest
public void testNestedPrimaryKeyQuery() throws Exception {
final RecordMetaDataHook hook = metaData -> {
metaData.getRecordType("MyRecord").setPrimaryKey(concat(field("header").nest(field("path")), field("header").nest(field("rec_no"))));
};
try (FDBRecordContext context = openContext()) {
openRecordWithHeader(context, hook);
saveHeaderRecord(1, "a", 0, "able");
saveHeaderRecord(2, "a", 3, "baker");
commit(context);
}
RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.and(Query.field("header").matches(Query.field("path").equalsValue("a")), Query.field("header").matches(Query.field("rec_no").equalsValue(2L)))).build();
// Scan([[a, 2],[a, 2]])
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, scan(bounds(hasTupleString("[[a, 2],[a, 2]]"))));
assertEquals(1265534819, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(136710600, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(-1817343447, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
try (FDBRecordContext context = openContext()) {
openRecordWithHeader(context, hook);
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
assertTrue(result.hasNext());
TestRecordsWithHeaderProto.MyRecord record = parseMyRecord(result.get().getRecord());
assertEquals("baker", record.getStrValue());
assertFalse(cursor.getNext().hasNext());
}
TestHelpers.assertDiscardedNone(context);
}
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class FDBSortQueryIndexSelectionTest method sortWithoutIndex.
@EnumSource(SortWithoutIndexMode.class)
@ParameterizedTest(name = "sortWithoutIndex [mode = {0}]")
public void sortWithoutIndex(SortWithoutIndexMode mode) throws Exception {
final int numberOfRecordsToSave = 2000;
final int numberOfResultsToReturn = 5;
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
for (int i = 0; i < numberOfRecordsToSave; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
// Slightly larger prime.
recBuilder.setRecNo((2244 * i + 1649) % 2357);
recBuilder.setNumValue2(i);
recBuilder.setNumValue3Indexed(i % 5);
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").greaterThanOrEquals(2)).setSort(field("num_value_2"), true).build();
if (mode == SortWithoutIndexMode.DISALLOWED) {
assertThrows(RecordCoreException.class, () -> {
planner.plan(query);
});
return;
}
((RecordQueryPlanner) planner).setConfiguration(((RecordQueryPlanner) planner).getConfiguration().asBuilder().setAllowNonIndexSort(true).build());
// Index(MySimpleRecord$num_value_3_indexed [[2],>) ORDER BY num_value_2 DESC
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, sort(allOf(hasProperty("reverse", equalTo(true))), indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],>"))))));
assertEquals(256365917, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(172993081, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(748321565, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
// Skip + limit is enough to overflow memory buffer. Skips into middle of section.
final int skip = mode == SortWithoutIndexMode.FILE ? numberOfRecordsToSave / 2 + 1 : 0;
ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setSkip(skip).setReturnedRowLimit(numberOfResultsToReturn);
final PrimitiveIterator.OfInt sortedInts = IntStream.iterate(numberOfRecordsToSave - 1, i -> i - 1).filter(i -> i % 5 >= 2).skip(skip).limit(numberOfResultsToReturn).iterator();
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
timer.reset();
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, executeProperties.build())) {
while (true) {
RecordCursorResult<FDBQueriedRecord<Message>> next = cursor.getNext();
if (!next.hasNext()) {
break;
}
FDBQueriedRecord<Message> rec = next.get();
TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
myrec.mergeFrom(rec.getRecord());
assertTrue(sortedInts.hasNext());
assertEquals(sortedInts.nextInt(), myrec.getNumValue2());
}
}
assertFalse(sortedInts.hasNext());
assertDiscardedNone(context);
int countBeforeSorting = (numberOfRecordsToSave / 5) * 3;
if (mode == SortWithoutIndexMode.MEMORY) {
assertEquals(0, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.MEMORY_SORT_STORE_RECORD));
assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.MEMORY_SORT_LOAD_RECORD));
} else {
int nfiles = numberOfRecordsToSave / RecordQuerySortAdapter.DEFAULT_MAX_RECORD_COUNT_IN_MEMORY;
assertEquals(nfiles, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
assertEquals(nfiles - 1, timer.getCount(SortEvents.Events.FILE_SORT_MERGE_FILES));
assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.FILE_SORT_SAVE_RECORD));
assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.FILE_SORT_LOAD_RECORD));
assertEquals(skip / RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_SECTION));
assertEquals(skip % RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_RECORD));
assertThat(timer.getCount(SortEvents.Counts.FILE_SORT_FILE_BYTES), allOf(greaterThan(1000), lessThan(100000)));
}
}
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class GeophileQueryTest method testJoin.
@Test
@Tag(Tags.Slow)
public // So really this just shows that join results aren't wildly off.
void testJoin() throws Exception {
final RecordMetaDataHook hook = md -> {
md.setSplitLongRecords(true);
md.addIndex("City", CITY_LOCATION_INDEX);
md.addIndex("Country", COUNTRY_SHAPE_INDEX);
};
loadCities(hook, 500000);
loadCountries(hook, 500000);
loadCountryShapes(hook);
final GeophileSpatialIndexJoinPlan plan = new GeophileSpatialIndexJoinPlan("City$location", ScanComparisons.EMPTY, "Country$shape", ScanComparisons.EMPTY);
int[] stats = new int[4];
try (FDBRecordContext context = openContext()) {
openRecordStore(context, hook);
RecordCursor<Pair<FDBIndexedRecord<Message>, FDBIndexedRecord<Message>>> recordCursor = plan.execute(recordStore, EvaluationContext.EMPTY);
final GeometryFactory geometryFactory = new GeometryFactory();
final GeoJsonReader geoJsonReader = new GeoJsonReader(geometryFactory);
recordCursor.forEach(pair -> {
TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(pair.getLeft().getRecord());
TestRecordsGeoProto.Country.Builder countryBuilder = TestRecordsGeoProto.Country.newBuilder().mergeFrom(pair.getRight().getRecord());
Point cityLocation = geometryFactory.createPoint(new Coordinate(cityBuilder.getLocation().getLatitude(), cityBuilder.getLocation().getLongitude()));
Geometry countryShape;
try {
countryShape = GeophileSpatial.swapLatLong(geoJsonReader.read(countryBuilder.getShape()));
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
// A real join would need to do this because of Z-order join false positives.
boolean contained;
try {
contained = countryShape.contains(cityLocation);
} catch (TopologyException ex) {
stats[3]++;
return;
}
if (!contained) {
stats[2]++;
} else if (!countryBuilder.getCode().equals(cityBuilder.getCountry())) {
LOGGER.warn(String.format("Code does not match: %s for %s <> %s for %s", countryBuilder.getCode(), countryBuilder.getName(), cityBuilder.getCountry(), cityBuilder.getName()));
stats[1]++;
} else {
LOGGER.debug(String.format("%s: %s", countryBuilder.getName(), cityBuilder.getName()));
stats[0]++;
}
}).join();
commit(context);
} catch (CompletionException ex) {
// transaction_too_old
assertThat(Throwables.getRootCause(ex), allOf(Matchers.instanceOf(FDBException.class), hasProperty("code", equalTo(1007))));
} finally {
LOGGER.info(String.format("match = %d, no match = %d, no overlap = %d, invalid geometry = %d", stats[0], stats[1], stats[2], stats[3]));
}
}
use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method scanTypedRecords.
@Nonnull
public <M extends Message> RecordCursor<FDBStoredRecord<M>> scanTypedRecords(@Nonnull RecordSerializer<M> typedSerializer, @Nullable final Tuple low, @Nullable final Tuple high, @Nonnull final EndpointType lowEndpoint, @Nonnull final EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
final Subspace recordsSubspace = recordsSubspace();
final SplitHelper.SizeInfo sizeInfo = new SplitHelper.SizeInfo();
final RecordCursor<FDBRawRecord> rawRecords;
if (metaData.isSplitLongRecords()) {
RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
} else {
KeyValueCursor.Builder keyValuesBuilder = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint);
if (omitUnsplitRecordSuffix) {
rawRecords = keyValuesBuilder.setScanProperties(scanProperties).build().map(kv -> {
sizeInfo.set(kv);
Tuple primaryKey = SplitHelper.unpackKey(recordsSubspace, kv);
return new FDBRawRecord(primaryKey, kv.getValue(), null, sizeInfo);
});
} else {
final ScanProperties finalScanProperties = scanProperties.with(executeProperties -> {
final ExecuteProperties.Builder builder = executeProperties.toBuilder().clearTimeLimit().clearSkipAndAdjustLimit().clearState();
int returnedRowLimit = builder.getReturnedRowLimitOrMax();
if (returnedRowLimit != Integer.MAX_VALUE) {
// Adjust limit to twice the supplied limit in case there are versions in the records
builder.setReturnedRowLimit(2 * returnedRowLimit);
}
return builder.build();
});
rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValuesBuilder.setScanProperties(finalScanProperties).build(), useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
}
}
RecordCursor<FDBStoredRecord<M>> result = rawRecords.mapPipelined(rawRecord -> {
final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional;
if (useOldVersionFormat()) {
// Older format versions: do a separate read to get the version.
versionFutureOptional = loadRecordVersionAsync(rawRecord.getPrimaryKey(), scanProperties.getExecuteProperties().getIsolationLevel().isSnapshot());
} else {
// Newer format versions: the version is either in the record or it is not -- do not do another read.
versionFutureOptional = Optional.empty();
}
return deserializeRecord(typedSerializer, rawRecord, metaData, versionFutureOptional);
}, pipelineSizer.getPipelineSize(PipelineOperation.KEY_TO_RECORD));
return context.instrument(FDBStoreTimer.Events.SCAN_RECORDS, result);
}
Aggregations