Search in sources :

Example 56 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBSortQueryIndexSelectionTest method testComplexLimits1.

/**
 * Verify that reverse sorts can be implemented by a reverse index scan.
 */
@DualPlannerTest
public void testComplexLimits1() throws Exception {
    RecordMetaDataHook hook = complexQuerySetupHook();
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setSort(field("str_value_indexed"), true).build();
    // Index(MySimpleRecord$str_value_indexed <,> REVERSE)
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, indexScan(allOf(indexName("MySimpleRecord$str_value_indexed"), unbounded())));
    assertTrue(plan.isReverse(), "plan is reversed");
    assertEquals(324762955, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(19722195, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(19722195, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("odd", myrec.getStrValueIndexed());
                i += 1;
            }
        }
        assertEquals(10, i);
        assertDiscardedNone(context);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 57 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBSortQueryIndexSelectionTest method sortUniqueNull.

/**
 * Verify that sort with filter works with different null interpretation on unique index.
 */
@DualPlannerTest
void sortUniqueNull() {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, NULL_UNIQUE_HOOK);
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo((1096 * i + 722) % 1289);
            recBuilder.setNumValueUnique(i);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_unique").greaterThanOrEquals(20)).setSort(field("num_value_unique")).build();
    // Index(MySimpleRecord$num_value_unique [[20],>)
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, indexScan(allOf(indexName("MySimpleRecord$num_value_unique"), bounds(hasTupleString("[[20],>")))));
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(-535398101, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1799532339, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1088253993, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(-535398101, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1799532345, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1088253999, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, NULL_UNIQUE_HOOK);
        int i = 20;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals(i++, myrec.getNumValueUnique());
            }
        }
        assertEquals(100, i);
        assertDiscardedNone(context);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Example 58 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method unorderedIntersectionWithScanLimit.

@ParameterizedTest(name = "unorderedIntersectionWithScanLimit [fail = {0}]")
@BooleanSource
public void unorderedIntersectionWithScanLimit(boolean fail) throws Exception {
    // TODO: When there is an UnorderedIntersectionPlan (or whatever) add that to the unordered plans stream
    RecordQueryPlanner planner = new RecordQueryPlanner(simpleMetaData(NO_HOOK), new RecordStoreState(null, null));
    RecordQueryPlan leftPlan = planner.plan(RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").startsWith("ev")).build());
    RecordQueryPlan rightPlan = planner.plan(RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").lessThanOrEquals(1)).build());
    int maximumToScan;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        maximumToScan = recordStore.executeQuery(leftPlan).getCount().get() + recordStore.executeQuery(rightPlan).getCount().get();
    }
    for (int limit = 0; limit < 3 * maximumToScan; limit = 2 * limit + 1) {
        final int finalLimit = limit;
        Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> cursorFunction = (continuation) -> {
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(finalLimit).setFailOnScanLimitReached(fail).build();
            return ProbableIntersectionCursor.create(record -> record.getPrimaryKey().getItems(), Arrays.asList(leftContinuation -> leftPlan.execute(recordStore, EvaluationContext.EMPTY, leftContinuation, executeProperties), rightContinuation -> rightPlan.execute(recordStore, EvaluationContext.EMPTY, rightContinuation, executeProperties)), continuation, recordStore.getTimer());
        };
        assertNumberOfRecordsScanned(limit, cursorFunction, fail, "should" + (limit >= maximumToScan ? "not " : "") + " be limited by record scan limit");
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordCursorVisitor(com.apple.foundationdb.record.RecordCursorVisitor) Tuple(com.apple.foundationdb.tuple.Tuple) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) QueryPlan(com.apple.foundationdb.record.query.plan.plans.QueryPlan) TestInstance(org.junit.jupiter.api.TestInstance) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) BeforeAll(org.junit.jupiter.api.BeforeAll) Tag(org.junit.jupiter.api.Tag) MethodSource(org.junit.jupiter.params.provider.MethodSource) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ScanLimitReachedException(com.apple.foundationdb.record.ScanLimitReachedException) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Arguments(org.junit.jupiter.params.provider.Arguments) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) List(java.util.List) Stream(java.util.stream.Stream) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Optional(java.util.Optional) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) ProbableIntersectionCursor(com.apple.foundationdb.record.provider.foundationdb.cursors.ProbableIntersectionCursor) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) RecordQueryPlanWithNoChildren(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanWithNoChildren) IndexScanParameters(com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Strings(com.google.common.base.Strings) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) IndexScanComparisons(com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons) Matchers.lessThan(org.hamcrest.Matchers.lessThan) BooleanSource(com.apple.test.BooleanSource) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) ValueSource(org.junit.jupiter.params.provider.ValueSource) RecordQueryIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Logger(org.slf4j.Logger) Tags(com.apple.test.Tags) BaseCursor(com.apple.foundationdb.record.cursors.BaseCursor) SplitHelper(com.apple.foundationdb.record.provider.foundationdb.SplitHelper) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) RecordCursor(com.apple.foundationdb.record.RecordCursor) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) BooleanSource(com.apple.test.BooleanSource)

Example 59 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method assertNumberOfRecordsScanned.

private void assertNumberOfRecordsScanned(int expected, Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> cursorFunction, boolean failOnLimitReached, String message) throws Exception {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        if (context.getTimer() != null) {
            context.getTimer().reset();
        }
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = cursorFunction.apply(null)) {
            boolean caughtScanLimitReached = false;
            RecordCursorResult<FDBQueriedRecord<Message>> result = null;
            try {
                do {
                    result = cursor.getNext();
                } while (result.hasNext());
            } catch (RecordCoreException ex) {
                if (failOnLimitReached && ex.getCause() instanceof ScanLimitReachedException) {
                    caughtScanLimitReached = true;
                } else {
                    throw ex;
                }
            }
            if (failOnLimitReached && !caughtScanLimitReached) {
                assertNotEquals(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, result.getNoNextReason());
            }
            Optional<Integer> scanned = getRecordScanned(context);
            if (context.getTimer() != null) {
                context.getTimer().reset();
            }
            int overrun = BaseCursorCountVisitor.getCount(cursor);
            scanned.ifPresent(value -> assertThat(message, value, lessThanOrEqualTo(expected + overrun)));
        }
    }
}
Also used : RecordCoreException(com.apple.foundationdb.record.RecordCoreException) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanLimitReachedException(com.apple.foundationdb.record.ScanLimitReachedException)

Example 60 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method testExecuteStateReset.

@ParameterizedTest
// for this test, the scan limit must divide 100
@ValueSource(ints = { 2, 5, 10, 20 })
public void testExecuteStateReset(int scanLimit) throws Exception {
    final IndexScanParameters fullValueScan = IndexScanComparisons.byValue();
    final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false);
    ExecuteProperties properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        byte[] continuation = null;
        do {
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, properties).asIterator()) {
                int retrieved = 0;
                while (cursor.hasNext()) {
                    cursor.next();
                    retrieved++;
                }
                continuation = cursor.getContinuation();
                if (continuation != null) {
                    // if this is our last call, we might retrieve 0 results
                    assertEquals(scanLimit, retrieved);
                }
                properties = properties.resetState();
            }
        } while (continuation != null);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) IndexScanParameters(com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQueryIndexPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

FDBQueriedRecord (com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord)137 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)125 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)117 RecordQuery (com.apple.foundationdb.record.query.RecordQuery)116 Message (com.google.protobuf.Message)108 Test (org.junit.jupiter.api.Test)65 Tags (com.apple.test.Tags)52 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)52 Tag (org.junit.jupiter.api.Tag)52 RecordQueryPlanner (com.apple.foundationdb.record.query.plan.RecordQueryPlanner)50 Query (com.apple.foundationdb.record.query.expressions.Query)47 List (java.util.List)47 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)46 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)45 Index (com.apple.foundationdb.record.metadata.Index)45 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)45 ArrayList (java.util.ArrayList)43 Collections (java.util.Collections)42 PlanHashable (com.apple.foundationdb.record.PlanHashable)41 Tuple (com.apple.foundationdb.tuple.Tuple)41