Search in sources :

Example 51 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectoryTest method testListAcrossTransactions.

@Test
public void testListAcrossTransactions() throws Exception {
    KeySpace root = new KeySpace(new KeySpaceDirectory("a", KeyType.LONG, random.nextLong()).addSubdirectory(new KeySpaceDirectory("b", KeyType.STRING)));
    final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    final List<String> directoryEntries = IntStream.range(0, 10).boxed().map(i -> "val_" + i).collect(Collectors.toList());
    final KeySpacePath rootPath = root.path("a");
    try (final FDBRecordContext context = database.openContext()) {
        final Transaction tr = context.ensureActive();
        directoryEntries.forEach(name -> tr.set(rootPath.add("b", name).toTuple(context).pack(), TupleHelpers.EMPTY.pack()));
        context.commit();
    }
    byte[] continuation = null;
    int idx = 0;
    do {
        try (final FDBRecordContext context = database.openContext()) {
            final RecordCursor<ResolvedKeySpacePath> cursor = rootPath.listSubdirectoryAsync(context, "b", continuation, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()));
            List<ResolvedKeySpacePath> subdirs = context.asyncToSync(FDBStoreTimer.Waits.WAIT_KEYSPACE_LIST, cursor.asList());
            if (!subdirs.isEmpty()) {
                assertEquals(2, subdirs.size(), "Wrong number of path entries returned");
                assertEquals("val_" + idx, subdirs.get(0).getResolvedValue());
                assertEquals("val_" + (idx + 1), subdirs.get(1).getResolvedValue());
                idx += 2;
                continuation = cursor.getNext().getContinuation().toBytes();
                System.out.println(continuation == null ? "null" : Tuple.fromBytes(continuation));
            } else {
                continuation = cursor.getNext().getContinuation().toBytes();
                assertNull(continuation);
            }
        }
    } while (continuation != null);
    assertEquals(directoryEntries.size(), idx);
}
Also used : Arrays(java.util.Arrays) IsInstanceOf.instanceOf(org.hamcrest.core.IsInstanceOf.instanceOf) BiFunction(java.util.function.BiFunction) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Tag(org.junit.jupiter.api.Tag) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) FDBDatabase(com.apple.foundationdb.record.provider.foundationdb.FDBDatabase) UUID(java.util.UUID) Assertions.assertNotSame(org.junit.jupiter.api.Assertions.assertNotSame) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) Matchers.startsWith(org.hamcrest.Matchers.startsWith) Test(org.junit.jupiter.api.Test) List(java.util.List) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) RandomStringUtils(org.apache.commons.lang3.RandomStringUtils) Matchers.is(org.hamcrest.Matchers.is) DEFAULT_CHECK(com.apple.foundationdb.record.provider.foundationdb.keyspace.ResolverCreateHooks.DEFAULT_CHECK) IntStream(java.util.stream.IntStream) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) Supplier(java.util.function.Supplier) FDBTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBTestBase) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) KeyType(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType) TestHelpers.eventually(com.apple.foundationdb.record.TestHelpers.eventually) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) LinkedList(java.util.LinkedList) NoSuchElementException(java.util.NoSuchElementException) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) Iterator(java.util.Iterator) Tags(com.apple.test.Tags) ScopedInterningLayer(com.apple.foundationdb.record.provider.foundationdb.layers.interning.ScopedInterningLayer) TimeUnit(java.util.concurrent.TimeUnit) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) TestHelpers.assertThrows(com.apple.foundationdb.record.TestHelpers.assertThrows) FDBDatabaseFactory(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory) RecordCursor(com.apple.foundationdb.record.RecordCursor) ValueRange(com.apple.foundationdb.record.ValueRange) FDBDatabase(com.apple.foundationdb.record.provider.foundationdb.FDBDatabase) Transaction(com.apple.foundationdb.Transaction) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanProperties(com.apple.foundationdb.record.ScanProperties) Test(org.junit.jupiter.api.Test)

Example 52 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method testNestedPrimaryKeyQuery.

/**
 * Verify that record scans with nested primary keys works properly.
 * Specifically, verify that a filter is implemented as a record scan in the case where there is a two-field
 * primary key both of whose fields are nested in some header subrecord.
 */
@DualPlannerTest
public void testNestedPrimaryKeyQuery() throws Exception {
    final RecordMetaDataHook hook = metaData -> {
        metaData.getRecordType("MyRecord").setPrimaryKey(concat(field("header").nest(field("path")), field("header").nest(field("rec_no"))));
    };
    try (FDBRecordContext context = openContext()) {
        openRecordWithHeader(context, hook);
        saveHeaderRecord(1, "a", 0, "able");
        saveHeaderRecord(2, "a", 3, "baker");
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.and(Query.field("header").matches(Query.field("path").equalsValue("a")), Query.field("header").matches(Query.field("rec_no").equalsValue(2L)))).build();
    // Scan([[a, 2],[a, 2]])
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, scan(bounds(hasTupleString("[[a, 2],[a, 2]]"))));
    assertEquals(1265534819, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(136710600, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1817343447, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openRecordWithHeader(context, hook);
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            assertTrue(result.hasNext());
            TestRecordsWithHeaderProto.MyRecord record = parseMyRecord(result.get().getRecord());
            assertEquals("baker", record.getStrValue());
            assertFalse(cursor.getNext().hasNext());
        }
        TestHelpers.assertDiscardedNone(context);
    }
}
Also used : Arrays(java.util.Arrays) TestRecords3Proto(com.apple.foundationdb.record.TestRecords3Proto) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) TestHelpers(com.apple.foundationdb.record.TestHelpers) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) PlanMatchers.indexScanType(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScanType) TestRecords4Proto(com.apple.foundationdb.record.TestRecords4Proto) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Matchers.allOf(org.hamcrest.Matchers.allOf) Test(org.junit.jupiter.api.Test) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) TestRecordsWithHeaderProto(com.apple.foundationdb.record.TestRecordsWithHeaderProto) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) QueryRecordFunction(com.apple.foundationdb.record.query.expressions.QueryRecordFunction) PlanMatchers.primaryKeyDistinct(com.apple.foundationdb.record.query.plan.match.PlanMatchers.primaryKeyDistinct) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) PlanMatchers.queryPredicateDescendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.queryPredicateDescendant) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) TestRecordsNestedMapProto(com.apple.foundationdb.record.TestRecordsNestedMapProto) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) Key(com.apple.foundationdb.record.metadata.Key) PlanMatchers.scan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.scan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) PredicateMatchers(com.apple.foundationdb.record.query.predicates.match.PredicateMatchers) TestRecords5Proto(com.apple.foundationdb.record.TestRecords5Proto) Index(com.apple.foundationdb.record.metadata.Index) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Collections(java.util.Collections) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) TestRecordsWithHeaderProto(com.apple.foundationdb.record.TestRecordsWithHeaderProto) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 53 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class FDBSortQueryIndexSelectionTest method sortWithoutIndex.

@EnumSource(SortWithoutIndexMode.class)
@ParameterizedTest(name = "sortWithoutIndex [mode = {0}]")
public void sortWithoutIndex(SortWithoutIndexMode mode) throws Exception {
    final int numberOfRecordsToSave = 2000;
    final int numberOfResultsToReturn = 5;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int i = 0; i < numberOfRecordsToSave; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            // Slightly larger prime.
            recBuilder.setRecNo((2244 * i + 1649) % 2357);
            recBuilder.setNumValue2(i);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").greaterThanOrEquals(2)).setSort(field("num_value_2"), true).build();
    if (mode == SortWithoutIndexMode.DISALLOWED) {
        assertThrows(RecordCoreException.class, () -> {
            planner.plan(query);
        });
        return;
    }
    ((RecordQueryPlanner) planner).setConfiguration(((RecordQueryPlanner) planner).getConfiguration().asBuilder().setAllowNonIndexSort(true).build());
    // Index(MySimpleRecord$num_value_3_indexed [[2],>) ORDER BY num_value_2 DESC
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, sort(allOf(hasProperty("reverse", equalTo(true))), indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],>"))))));
    assertEquals(256365917, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(172993081, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(748321565, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    // Skip + limit is enough to overflow memory buffer. Skips into middle of section.
    final int skip = mode == SortWithoutIndexMode.FILE ? numberOfRecordsToSave / 2 + 1 : 0;
    ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setSkip(skip).setReturnedRowLimit(numberOfResultsToReturn);
    final PrimitiveIterator.OfInt sortedInts = IntStream.iterate(numberOfRecordsToSave - 1, i -> i - 1).filter(i -> i % 5 >= 2).skip(skip).limit(numberOfResultsToReturn).iterator();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        timer.reset();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, executeProperties.build())) {
            while (true) {
                RecordCursorResult<FDBQueriedRecord<Message>> next = cursor.getNext();
                if (!next.hasNext()) {
                    break;
                }
                FDBQueriedRecord<Message> rec = next.get();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertTrue(sortedInts.hasNext());
                assertEquals(sortedInts.nextInt(), myrec.getNumValue2());
            }
        }
        assertFalse(sortedInts.hasNext());
        assertDiscardedNone(context);
        int countBeforeSorting = (numberOfRecordsToSave / 5) * 3;
        if (mode == SortWithoutIndexMode.MEMORY) {
            assertEquals(0, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
            assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.MEMORY_SORT_STORE_RECORD));
            assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.MEMORY_SORT_LOAD_RECORD));
        } else {
            int nfiles = numberOfRecordsToSave / RecordQuerySortAdapter.DEFAULT_MAX_RECORD_COUNT_IN_MEMORY;
            assertEquals(nfiles, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
            assertEquals(nfiles - 1, timer.getCount(SortEvents.Events.FILE_SORT_MERGE_FILES));
            assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.FILE_SORT_SAVE_RECORD));
            assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.FILE_SORT_LOAD_RECORD));
            assertEquals(skip / RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_SECTION));
            assertEquals(skip % RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_RECORD));
            assertThat(timer.getCount(SortEvents.Counts.FILE_SORT_FILE_BYTES), allOf(greaterThan(1000), lessThan(100000)));
        }
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Arrays(java.util.Arrays) Matchers.not(org.hamcrest.Matchers.not) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) TestHelpers.assertDiscardedNone(com.apple.foundationdb.record.TestHelpers.assertDiscardedNone) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) TestHelpers(com.apple.foundationdb.record.TestHelpers) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) MethodSource(org.junit.jupiter.params.provider.MethodSource) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Matchers.allOf(org.hamcrest.Matchers.allOf) IndexMaintainerFactory(com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory) FanType(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType) Arguments(org.junit.jupiter.params.provider.Arguments) Sets(com.google.common.collect.Sets) Test(org.junit.jupiter.api.Test) Objects(java.util.Objects) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) Stream(java.util.stream.Stream) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Matchers.contains(org.hamcrest.Matchers.contains) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) TestRecordsWithHeaderProto(com.apple.foundationdb.record.TestRecordsWithHeaderProto) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) PlanMatchers.typeFilter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.typeFilter) IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) EnumSource(org.junit.jupiter.params.provider.EnumSource) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) Key(com.apple.foundationdb.record.metadata.Key) Matchers.hasProperty(org.hamcrest.Matchers.hasProperty) TestHelpers.assertDiscardedExactly(com.apple.foundationdb.record.TestHelpers.assertDiscardedExactly) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) RecordQuerySortAdapter(com.apple.foundationdb.record.query.plan.sorting.RecordQuerySortAdapter) PlanMatchers.scan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.scan) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Matchers.lessThan(org.hamcrest.Matchers.lessThan) BooleanSource(com.apple.test.BooleanSource) ValueIndexMaintainerFactory(com.apple.foundationdb.record.provider.foundationdb.indexes.ValueIndexMaintainerFactory) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) PlannableIndexTypes(com.apple.foundationdb.record.query.plan.PlannableIndexTypes) SortEvents(com.apple.foundationdb.record.sorting.SortEvents) PrimitiveIterator(java.util.PrimitiveIterator) AtomicLong(java.util.concurrent.atomic.AtomicLong) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) TestHelpers.assertDiscardedAtMost(com.apple.foundationdb.record.TestHelpers.assertDiscardedAtMost) PlanMatchers.unbounded(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unbounded) AutoService(com.google.auto.service.AutoService) Matcher(org.hamcrest.Matcher) PlanMatchers.sort(com.apple.foundationdb.record.query.plan.match.PlanMatchers.sort) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Collections(java.util.Collections) PrimitiveIterator(java.util.PrimitiveIterator) Message(com.google.protobuf.Message) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) EnumSource(org.junit.jupiter.params.provider.EnumSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 54 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class GeophileQueryTest method testJoin.

@Test
@Tag(Tags.Slow)
public // So really this just shows that join results aren't wildly off.
void testJoin() throws Exception {
    final RecordMetaDataHook hook = md -> {
        md.setSplitLongRecords(true);
        md.addIndex("City", CITY_LOCATION_INDEX);
        md.addIndex("Country", COUNTRY_SHAPE_INDEX);
    };
    loadCities(hook, 500000);
    loadCountries(hook, 500000);
    loadCountryShapes(hook);
    final GeophileSpatialIndexJoinPlan plan = new GeophileSpatialIndexJoinPlan("City$location", ScanComparisons.EMPTY, "Country$shape", ScanComparisons.EMPTY);
    int[] stats = new int[4];
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<Pair<FDBIndexedRecord<Message>, FDBIndexedRecord<Message>>> recordCursor = plan.execute(recordStore, EvaluationContext.EMPTY);
        final GeometryFactory geometryFactory = new GeometryFactory();
        final GeoJsonReader geoJsonReader = new GeoJsonReader(geometryFactory);
        recordCursor.forEach(pair -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(pair.getLeft().getRecord());
            TestRecordsGeoProto.Country.Builder countryBuilder = TestRecordsGeoProto.Country.newBuilder().mergeFrom(pair.getRight().getRecord());
            Point cityLocation = geometryFactory.createPoint(new Coordinate(cityBuilder.getLocation().getLatitude(), cityBuilder.getLocation().getLongitude()));
            Geometry countryShape;
            try {
                countryShape = GeophileSpatial.swapLatLong(geoJsonReader.read(countryBuilder.getShape()));
            } catch (ParseException ex) {
                throw new RuntimeException(ex);
            }
            // A real join would need to do this because of Z-order join false positives.
            boolean contained;
            try {
                contained = countryShape.contains(cityLocation);
            } catch (TopologyException ex) {
                stats[3]++;
                return;
            }
            if (!contained) {
                stats[2]++;
            } else if (!countryBuilder.getCode().equals(cityBuilder.getCountry())) {
                LOGGER.warn(String.format("Code does not match: %s for %s <> %s for %s", countryBuilder.getCode(), countryBuilder.getName(), cityBuilder.getCountry(), cityBuilder.getName()));
                stats[1]++;
            } else {
                LOGGER.debug(String.format("%s: %s", countryBuilder.getName(), cityBuilder.getName()));
                stats[0]++;
            }
        }).join();
        commit(context);
    } catch (CompletionException ex) {
        // transaction_too_old
        assertThat(Throwables.getRootCause(ex), allOf(Matchers.instanceOf(FDBException.class), hasProperty("code", equalTo(1007))));
    } finally {
        LOGGER.info(String.format("match = %d, no match = %d, no overlap = %d, invalid geometry = %d", stats[0], stats[1], stats[2], stats[3]));
    }
}
Also used : RecordQueryFilterPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryFilterPlan) LoggerFactory(org.slf4j.LoggerFactory) Coordinate(org.locationtech.jts.geom.Coordinate) Bindings(com.apple.foundationdb.record.Bindings) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Tuple(com.apple.foundationdb.tuple.Tuple) Pair(org.apache.commons.lang3.tuple.Pair) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) RecordQueryScanPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) Query(com.apple.foundationdb.record.query.expressions.Query) DoubleValueOrParameter(com.apple.foundationdb.record.spatial.common.DoubleValueOrParameter) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) Point(org.locationtech.jts.geom.Point) CompletionException(java.util.concurrent.CompletionException) Expressions.keyWithValue(com.apple.foundationdb.record.metadata.Key.Expressions.keyWithValue) Test(org.junit.jupiter.api.Test) GeoJsonReader(org.locationtech.jts.io.geojson.GeoJsonReader) TopologyException(org.locationtech.jts.geom.TopologyException) ParseException(org.locationtech.jts.io.ParseException) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Geometry(org.locationtech.jts.geom.Geometry) Expressions.function(com.apple.foundationdb.record.metadata.Key.Expressions.function) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) RecordQueryTypeFilterPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan) GeoPointWithinDistanceComponent(com.apple.foundationdb.record.spatial.common.GeoPointWithinDistanceComponent) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) FDBRecordStoreQueryTestBase(com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) HashSet(java.util.HashSet) Matchers.hasProperty(org.hamcrest.Matchers.hasProperty) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) GeometryFactory(org.locationtech.jts.geom.GeometryFactory) Logger(org.slf4j.Logger) Tags(com.apple.test.Tags) Throwables(com.google.common.base.Throwables) Matchers(org.hamcrest.Matchers) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) FileInputStream(java.io.FileInputStream) InputStreamReader(java.io.InputStreamReader) Index(com.apple.foundationdb.record.metadata.Index) Expressions.value(com.apple.foundationdb.record.metadata.Key.Expressions.value) FDBException(com.apple.foundationdb.FDBException) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) BufferedReader(java.io.BufferedReader) Collections(java.util.Collections) GeometryFactory(org.locationtech.jts.geom.GeometryFactory) Message(com.google.protobuf.Message) FDBException(com.apple.foundationdb.FDBException) Point(org.locationtech.jts.geom.Point) TopologyException(org.locationtech.jts.geom.TopologyException) Geometry(org.locationtech.jts.geom.Geometry) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) GeoJsonReader(org.locationtech.jts.io.geojson.GeoJsonReader) Coordinate(org.locationtech.jts.geom.Coordinate) CompletionException(java.util.concurrent.CompletionException) ParseException(org.locationtech.jts.io.ParseException) Pair(org.apache.commons.lang3.tuple.Pair) Test(org.junit.jupiter.api.Test) Tag(org.junit.jupiter.api.Tag)

Example 55 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method scanTypedRecords.

@Nonnull
public <M extends Message> RecordCursor<FDBStoredRecord<M>> scanTypedRecords(@Nonnull RecordSerializer<M> typedSerializer, @Nullable final Tuple low, @Nullable final Tuple high, @Nonnull final EndpointType lowEndpoint, @Nonnull final EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
    final Subspace recordsSubspace = recordsSubspace();
    final SplitHelper.SizeInfo sizeInfo = new SplitHelper.SizeInfo();
    final RecordCursor<FDBRawRecord> rawRecords;
    if (metaData.isSplitLongRecords()) {
        RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
        rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
    } else {
        KeyValueCursor.Builder keyValuesBuilder = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint);
        if (omitUnsplitRecordSuffix) {
            rawRecords = keyValuesBuilder.setScanProperties(scanProperties).build().map(kv -> {
                sizeInfo.set(kv);
                Tuple primaryKey = SplitHelper.unpackKey(recordsSubspace, kv);
                return new FDBRawRecord(primaryKey, kv.getValue(), null, sizeInfo);
            });
        } else {
            final ScanProperties finalScanProperties = scanProperties.with(executeProperties -> {
                final ExecuteProperties.Builder builder = executeProperties.toBuilder().clearTimeLimit().clearSkipAndAdjustLimit().clearState();
                int returnedRowLimit = builder.getReturnedRowLimitOrMax();
                if (returnedRowLimit != Integer.MAX_VALUE) {
                    // Adjust limit to twice the supplied limit in case there are versions in the records
                    builder.setReturnedRowLimit(2 * returnedRowLimit);
                }
                return builder.build();
            });
            rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValuesBuilder.setScanProperties(finalScanProperties).build(), useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
        }
    }
    RecordCursor<FDBStoredRecord<M>> result = rawRecords.mapPipelined(rawRecord -> {
        final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional;
        if (useOldVersionFormat()) {
            // Older format versions: do a separate read to get the version.
            versionFutureOptional = loadRecordVersionAsync(rawRecord.getPrimaryKey(), scanProperties.getExecuteProperties().getIsolationLevel().isSnapshot());
        } else {
            // Newer format versions: the version is either in the record or it is not -- do not do another read.
            versionFutureOptional = Optional.empty();
        }
        return deserializeRecord(typedSerializer, rawRecord, metaData, versionFutureOptional);
    }, pipelineSizer.getPipelineSize(PipelineOperation.KEY_TO_RECORD));
    return context.instrument(FDBStoreTimer.Events.SCAN_RECORDS, result);
}
Also used : LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) UnaryOperator(java.util.function.UnaryOperator) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordSerializer(com.apple.foundationdb.record.provider.common.RecordSerializer) Subspace(com.apple.foundationdb.subspace.Subspace) MutationType(com.apple.foundationdb.MutationType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Map(java.util.Map) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Set(java.util.Set) TupleRange(com.apple.foundationdb.record.TupleRange) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) ByteOrder(java.nio.ByteOrder) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RangeSet(com.apple.foundationdb.async.RangeSet) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Supplier(java.util.function.Supplier) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) ArrayList(java.util.ArrayList) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) ParameterRelationshipGraph(com.apple.foundationdb.record.query.ParameterRelationshipGraph) LoggableException(com.apple.foundationdb.util.LoggableException) CloseableAsyncIterator(com.apple.foundationdb.async.CloseableAsyncIterator) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) CursorLimitManager(com.apple.foundationdb.record.cursors.CursorLimitManager) ExecuteState(com.apple.foundationdb.record.ExecuteState) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) DynamicMessageRecordSerializer(com.apple.foundationdb.record.provider.common.DynamicMessageRecordSerializer) SyntheticRecordPlanner(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner) IndexEntry(com.apple.foundationdb.record.IndexEntry) LoggerFactory(org.slf4j.LoggerFactory) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) ByteBuffer(java.nio.ByteBuffer) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordMetaDataProto(com.apple.foundationdb.record.RecordMetaDataProto) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) AndComponent(com.apple.foundationdb.record.query.expressions.AndComponent) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) AggregateFunctionNotSupportedException(com.apple.foundationdb.record.AggregateFunctionNotSupportedException) RecordTypeKeyComparison(com.apple.foundationdb.record.query.expressions.RecordTypeKeyComparison) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) MutableRecordStoreState(com.apple.foundationdb.record.MutableRecordStoreState) RecordTypeOrBuilder(com.apple.foundationdb.record.metadata.RecordTypeOrBuilder) SyntheticRecordFromStoredRecordPlan(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordFromStoredRecordPlan) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings) Descriptors(com.google.protobuf.Descriptors) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) CursorStreamingMode(com.apple.foundationdb.record.CursorStreamingMode) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) Suppliers(com.google.common.base.Suppliers) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) IndexState(com.apple.foundationdb.record.IndexState) StoreRecordFunction(com.apple.foundationdb.record.metadata.StoreRecordFunction) ReadTransaction(com.apple.foundationdb.ReadTransaction) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) FDBRecordStoreStateCache(com.apple.foundationdb.record.provider.foundationdb.storestate.FDBRecordStoreStateCache) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) KeyValue(com.apple.foundationdb.KeyValue) CursorLimitManager(com.apple.foundationdb.record.cursors.CursorLimitManager) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) CompletableFuture(java.util.concurrent.CompletableFuture) ScanProperties(com.apple.foundationdb.record.ScanProperties) Subspace(com.apple.foundationdb.subspace.Subspace) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Aggregations

RecordCursor (com.apple.foundationdb.record.RecordCursor)66 List (java.util.List)59 Nonnull (javax.annotation.Nonnull)57 Message (com.google.protobuf.Message)50 Collections (java.util.Collections)48 Function (java.util.function.Function)48 ArrayList (java.util.ArrayList)47 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)46 Tuple (com.apple.foundationdb.tuple.Tuple)44 Test (org.junit.jupiter.api.Test)43 ScanProperties (com.apple.foundationdb.record.ScanProperties)42 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)40 RecordCursorResult (com.apple.foundationdb.record.RecordCursorResult)39 Tags (com.apple.test.Tags)37 Collectors (java.util.stream.Collectors)37 Tag (org.junit.jupiter.api.Tag)37 Arrays (java.util.Arrays)36 Nullable (javax.annotation.Nullable)35 Index (com.apple.foundationdb.record.metadata.Index)34 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)33