Examples with FDBQueriedRecord com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord used on opensource projects

Example 86 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method queryWithWideOrOfFullTextPrefixPredicates.

/**
 * Queries with an OR of {@link com.apple.foundationdb.record.query.expressions.Text#containsPrefix(String)}
 * predicates get planned as {@link com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan}s,
 * which have unusual semantics where results are returned in an undefined order as soon as any child has one.
 * Therefore, the assertions made in {@link #assertPlanLimitsWithCorrectExecution(List, FDBRecordContext, RecordQueryPlan)}
 * are far too strong for plans like this. Instead, we make very weak assertions that the byte scan limit does
 * <em>something</em>.
 */
@ParameterizedTest
@MethodSource("complexTextQueries")
public void queryWithWideOrOfFullTextPrefixPredicates(@Nonnull RecordQuery query, int numPredicates) throws Exception {
    deleteSimpleRecords();
    final List<String> textSamples = ImmutableList.of(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH, TextSamples.KOREAN);
    RecordMetaDataHook indexHook = metaDataBuilder -> metaDataBuilder.addIndex(metaDataBuilder.getRecordType(SIMPLE_DOC), SIMPLE_TEXT_PREFIX);
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        for (int i = 0; i < textSamples.size(); i++) {
            recordStore.saveRecord(TestRecordsTextProto.SimpleDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).build());
        }
        commit(context);
    }
    setupPlanner(null);
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, descendant(unorderedUnion(Collections.nCopies(numPredicates, any(RecordQueryPlan.class)))));
    long totalBytes;
    Set<Long> noLimitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        context.getTimer().reset();
        RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, null, ExecuteProperties.SERIAL_EXECUTE);
        RecordCursorResult<FDBQueriedRecord<Message>> result;
        do {
            result = cursor.onNext().get();
            if (result.hasNext()) {
                TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                record.mergeFrom(result.get().getRecord());
                noLimitRecordIds.add(record.getDocId());
            }
        } while (result.hasNext());
        totalBytes = byteCounter.getBytesScanned(context);
    }
    Set<Long> limitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context);
        ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(0);
        byte[] continuation = null;
        do {
            context.getTimer().reset();
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, continuation, executeProperties.build());
            RecordCursorResult<FDBQueriedRecord<Message>> result;
            do {
                result = cursor.onNext().get();
                if (result.hasNext()) {
                    TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                    record.mergeFrom(result.get().getRecord());
                    limitRecordIds.add(record.getDocId());
                }
            } while (result.hasNext());
            assertThat(byteCounter.getBytesScanned(context), lessThan(totalBytes));
            continuation = result.getContinuation().toBytes();
            if (continuation != null) {
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
        } while (continuation != null);
        assertEquals(noLimitRecordIds, limitRecordIds);
    }
}

Example 87 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method plansByContinuation.

@ParameterizedTest(name = "plansByContinuation() [{index}] {0}")
@MethodSource("plansWithoutFail")
public void plansByContinuation(String description, boolean fail, RecordQueryPlan plan) throws Exception {
    int maximumToScan = getMaximumToScan(plan);
    // include a scanLimit of 0, in which case all progress happens via the first "free" key-value scan.
    for (int scanLimit = 0; scanLimit <= maximumToScan * 2; scanLimit = 2 * scanLimit + 1) {
        final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
            TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
            record.mergeFrom(r.getRecord());
            return record.getRecNo();
        };
        final ExecuteProperties.Builder properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit);
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context);
            final List<Long> allAtOnce;
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
                allAtOnce = cursor.map(getRecNo).asList().get();
            }
            final List<Long> byContinuation = new ArrayList<>();
            byte[] continuation = null;
            do {
                try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, properties.build())) {
                    if (context.getTimer() != null) {
                        context.getTimer().reset();
                    }
                    RecordCursorResult<FDBQueriedRecord<Message>> result;
                    while ((result = cursor.getNext()).hasNext()) {
                        byContinuation.add(getRecNo.apply(result.get()));
                    }
                    continuation = result.getContinuation().toBytes();
                    int overrun = BaseCursorCountVisitor.getCount(cursor);
                    Optional<Integer> recordScanned = getRecordScanned(context);
                    if (recordScanned.isPresent()) {
                        assertThat(recordScanned.get(), lessThanOrEqualTo(Math.min(scanLimit + overrun, maximumToScan)));
                    }
                }
            } while (continuation != null);
            assertEquals(allAtOnce, byContinuation);
        }
    }
}

Example 88 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method testWithFailOnByteScanLimitReached.

@ParameterizedTest(name = "testWithFailOnByteScanLimitReached [{index}] {0} {1}")
@MethodSource("plans")
public void testWithFailOnByteScanLimitReached(String description, boolean notUsed, @Nonnull RecordQueryPlan plan) throws Exception {
    setupSimpleRecordStore();
    for (long byteLimit = 0; byteLimit < 1000; byteLimit += 100) {
        try (FDBRecordContext context = openContext()) {
            ExecuteProperties properties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteLimit).setFailOnScanLimitReached(true).build();
            openSimpleRecordStore(context);
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, properties);
            assertThrowsWithWrapper(ScanLimitReachedException.class, () -> cursor.asList().join());
        }
    }
}

Example 89 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQueryOrdered.

/**
 * Verify that an OR query on the same field with a sort on that field is implemented as a union of index scans,
 * where the union ordering is the field (and not the primary key, as it would normally be for equality predicates).
 * TODO The planner could be smarter here:
 * TODO: Add RecordQueryConcatenationPlan for non-overlapping unions (https://github.com/FoundationDB/fdb-record-layer/issues/13)
 * Note that the ordering planner property evaluation now understands that num_value_3_indexed is equality-bound
 * and does not need to partake in the ordering.
 */
@DualPlannerTest
void testOrQueryOrdered() throws Exception {
    RecordMetaDataHook hook = complexPrimaryKeyHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("num_value_3_indexed").equalsValue(1), Query.field("num_value_3_indexed").equalsValue(3))).setSort(field("num_value_3_indexed")).build();
    // Index(MySimpleRecord$num_value_3_indexed [[1],[1]]) ∪[Field { 'num_value_3_indexed' None}, Field { 'str_value_indexed' None}, Field { 'num_value_unique' None}] Index(MySimpleRecord$num_value_3_indexed [[3],[3]])
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1412961915, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(258619931, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1414232579, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1300798826, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1882806542, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(739308244, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                int numValue3 = myrec.getNumValue3Indexed();
                assertTrue(numValue3 == 1 || numValue3 == 3, "should satisfy value condition");
                assertTrue(numValue3 == 1 || i >= 20, "lower values should come first");
                i++;
            }
        }
        assertEquals(20 + 20, i);
        assertDiscardedNone(context);
    }
    query = query.toBuilder().setSort(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"))).build();
    plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1412961915, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(258435419, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1414417091, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[1],[1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("[[3],[3]]")))))).where(comparisonKey(concat(field("num_value_3_indexed"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1300798826, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1882991054, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(739123732, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
}

Example 90 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQueryChildReordering2.

@DualPlannerTest
@ParameterizedTest
@BooleanSource
void testOrQueryChildReordering2(boolean shouldDeferFetch) throws Exception {
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    RecordQuery query1 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("str_value_indexed").equalsValue("odd"), Query.field("num_value_3_indexed").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(3))).setSort(null, true).setRemoveDuplicates(true).build();
    setDeferFetchAfterUnionAndIntersection(shouldDeferFetch);
    // Index(MySimpleRecord$str_value_indexed [[odd],[odd]] REVERSE) ∪ Index(MySimpleRecord$num_value_3_indexed [[0],[0]] REVERSE) ∪ Index(MySimpleRecord$num_value_3_indexed [[3],[3]] REVERSE)
    // Fetch(Covering(Index(MySimpleRecord$str_value_indexed [[odd],[odd]] REVERSE) -> [rec_no: KEY[1], str_value_indexed: KEY[0]]) ∪ Covering(Index(MySimpleRecord$num_value_3_indexed [[0],[0]] REVERSE) -> [num_value_3_indexed: KEY[0], rec_no: KEY[1]]) ∪ Covering(Index(MySimpleRecord$num_value_3_indexed [[3],[3]] REVERSE) -> [num_value_3_indexed: KEY[0], rec_no: KEY[1]]))
    RecordQueryPlan plan1 = planner.plan(query1);
    RecordQuery query2 = query1.toBuilder().setFilter(Query.or(Lists.reverse(Objects.requireNonNull((OrComponent) query1.getFilter()).getChildren()))).build();
    // Index(MySimpleRecord$num_value_3_indexed [[3],[3]] REVERSE) ∪ Index(MySimpleRecord$num_value_3_indexed [[0],[0]] REVERSE) ∪ Index(MySimpleRecord$str_value_indexed [[odd],[odd]] REVERSE)
    // Fetch(Covering(Index(MySimpleRecord$num_value_3_indexed [[3],[3]] REVERSE) -> [num_value_3_indexed: KEY[0], rec_no: KEY[1]]) ∪ Covering(Index(MySimpleRecord$num_value_3_indexed [[0],[0]] REVERSE) -> [num_value_3_indexed: KEY[0], rec_no: KEY[1]]) ∪ Covering(Index(MySimpleRecord$str_value_indexed [[odd],[odd]] REVERSE) -> [rec_no: KEY[1], str_value_indexed: KEY[0]]))
    RecordQueryPlan plan2 = planner.plan(query2);
    assertNotEquals(plan1.hashCode(), plan2.hashCode());
    assertNotEquals(plan1, plan2);
    assertEquals(plan1.semanticHashCode(), plan2.semanticHashCode());
    assertTrue(plan1.semanticEquals(plan2));
    if (shouldDeferFetch || planner instanceof CascadesPlanner) {
        assertEquals(770691035, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1890796442, plan1.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(55660884, plan1.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertEquals(1289607451, plan2.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-29394342, plan2.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1772831508, plan2.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(723665474, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-330673401, plan1.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(2129158337, plan1.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertEquals(184229634, plan2.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(2044103111, plan2.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-448638335, plan2.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    Set<Long> seen = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor1 = recordStore.executeQuery(plan1).asIterator();
            RecordCursorIterator<FDBQueriedRecord<Message>> cursor2 = recordStore.executeQuery(plan2).asIterator()) {
            while (cursor1.hasNext()) {
                assertThat(cursor2.hasNext(), is(true));
                FDBQueriedRecord<Message> rec1 = cursor1.next();
                FDBQueriedRecord<Message> rec2 = cursor2.next();
                assertEquals(Objects.requireNonNull(rec1).getRecord(), Objects.requireNonNull(rec2).getRecord());
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(rec1.getRecord());
                assertTrue(myrec.getStrValueIndexed().equals("odd") || myrec.getNumValue3Indexed() == 0 || myrec.getNumValue3Indexed() == 3, "condition on record not met");
                assertFalse(seen.contains(myrec.getRecNo()), "Already saw a record!");
                seen.add(myrec.getRecNo());
                i++;
            }
            assertThat(cursor2.hasNext(), is(false));
        }
        assertEquals(70, i);
        assertDiscardedAtMost(40, context);
        if (shouldDeferFetch) {
            assertLoadRecord(140, context);
        }
    }
}