Examples with ExecuteProperties com.apple.foundationdb.record.ExecuteProperties used on opensource projects

Example 16 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method testExecuteStateReset.

@ParameterizedTest
// for this test, the scan limit must divide 100
@ValueSource(ints = { 2, 5, 10, 20 })
public void testExecuteStateReset(int scanLimit) throws Exception {
    final IndexScanParameters fullValueScan = IndexScanComparisons.byValue();
    final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false);
    ExecuteProperties properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        byte[] continuation = null;
        do {
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, properties).asIterator()) {
                int retrieved = 0;
                while (cursor.hasNext()) {
                    cursor.next();
                    retrieved++;
                }
                continuation = cursor.getContinuation();
                if (continuation != null) {
                    // if this is our last call, we might retrieve 0 results
                    assertEquals(scanLimit, retrieved);
                }
                properties = properties.resetState();
            }
        } while (continuation != null);
    }
}

Example 17 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method testPlansReturnSameRecordsRegardlessOfLimit.

@ParameterizedTest(name = "plansByContinuation() [{index}] {0}")
@MethodSource("plans")
public void testPlansReturnSameRecordsRegardlessOfLimit(String description, boolean notUsed, RecordQueryPlan plan) throws Exception {
    setupSimpleRecordStore();
    final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
        TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
        record.mergeFrom(r.getRecord());
        return record.getRecNo();
    };
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final List<Long> allAtOnce;
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            allAtOnce = cursor.map(getRecNo).asList().get();
        }
        for (long byteLimit = 0; byteLimit < 1000; byteLimit += 100) {
            final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteLimit).build();
            final List<Long> byContinuation = new ArrayList<>(allAtOnce.size());
            byte[] continuation = null;
            do {
                try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
                    RecordCursorResult<Long> result;
                    do {
                        result = cursor.onNext().get().map(getRecNo);
                        if (result.hasNext()) {
                            byContinuation.add(result.get());
                        }
                    } while (result.hasNext());
                    continuation = result.getContinuation().toBytes();
                }
            } while (continuation != null);
            assertEquals(allAtOnce, byContinuation);
        }
    }
}

Example 18 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method assertPlanLimitsWithCorrectExecution.

/**
 * Make detailed assertions about the number of bytes scanned when the byte scan limit is very close to the number
 * of records scanned between records.
 *
 * This helper method attempts to verify that the number of bytes being scanned during query execution is not too
 * much higher than the limit specified by {@link ExecuteProperties.Builder#setScannedBytesLimit(long)}. To do this,
 * it's given a list of the number of bytes scanned between individual records produced by the given plan. Then,
 * it sets the byte limit to that number of bytes, possibly increased or decreased by 1.
 *
 * Suppose that A, B, and C are successive records produced by the plan's cursor. Consider the execution of that
 * cursor in the absence of any limits. Let the number of bytes scanned between A and B be {@code m} and the number
 * of bytes scanned between B and C be {@code n}. This test relies on the following claims holding:
 * a) {@code m} and {@code n} are constant with respect to the asynchronous execution order of the underlying cursors.
 * b) If the byte scan limit is less than or equal to {@code m}, then at most {@code 2 * m} bytes are scanned between
 *   A and B, even if we resume the cursor by continuation. The same holds for {@code n}, B and C.
 * c) If the byte scan limit is {@code m + 1}, then at least {@code m} and at most {@code m + n} bytes are scanned.
 *
 * Note that claim (b) requires the factor of two because resuming certain cursors (such as a
 * {@link com.apple.foundationdb.record.provider.foundationdb.cursors.UnionCursor}) from a continuation requires
 * rescanning records that have already been deserialized.
 *
 * These claims should hold for most query plans. Notably, even claim (a) does not hold for the
 * {@link com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan}.
 * @param byteCountsByRecord a list of the number of bytes scanned between successive records without any limits
 * @param context an open record store context
 * @param plan a query plan to execute
 */
private void assertPlanLimitsWithCorrectExecution(@Nonnull List<Long> byteCountsByRecord, @Nonnull FDBRecordContext context, @Nonnull RecordQueryPlan plan) {
    byte[] continuation = null;
    int i = 0;
    context.getTimer().reset();
    while (i < byteCountsByRecord.size()) {
        final int currentIndex = i;
        // If the limit is slightly too low to scan the next record, we should scan it anyway and then stop.
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteCountsByRecord.get(currentIndex) - 1).build();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            final long bytesScanned = byteCounter.getBytesScanned(context);
            if (currentIndex == byteCountsByRecord.size() - 1) {
                // Final record
                // If this is a final record, then there must be enough room to scan everything until the end,
                // because of how the byteCountsByRecord are counted.
                assertTrue(result.hasNext());
            }
            // cursors) then we might not have a result when we expect to.
            if (result.hasNext()) {
                i++;
                context.getTimer().reset();
            } else {
                // Check that we stopped because of the byte scan limit.
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
            // Assertion of claim (b)
            assertThat(bytesScanned, lessThanOrEqualTo(2 * byteCountsByRecord.get(currentIndex)));
            continuation = result.getContinuation().toBytes();
        }
    }
    continuation = null;
    i = 0;
    while (i < byteCountsByRecord.size()) {
        final int currentIndex = i;
        // If the limit is exactly right we should scan the record and then stop.
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteCountsByRecord.get(currentIndex)).build();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            final long bytesScanned = byteCounter.getBytesScanned(context);
            // cursors) then we might not have a result when we expect to.
            if (result.hasNext()) {
                i++;
                context.getTimer().reset();
            } else if (currentIndex < byteCountsByRecord.size() - 1) {
                // not the final record yet
                // Check that we stopped because of the byte scan limit.
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
            // Assertion of claim (b)
            assertThat(bytesScanned, lessThanOrEqualTo(2 * byteCountsByRecord.get(currentIndex)));
            continuation = result.getContinuation().toBytes();
        }
    }
    continuation = null;
    i = 0;
    while (i < byteCountsByRecord.size()) {
        // If the limit is slightly too high we should scan the record, proceed to the next one (if it exists), and then stop.
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteCountsByRecord.get(i) + 1).build();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            assertTrue(result.hasNext());
            continuation = result.getContinuation().toBytes();
            result = cursor.getNext();
            long bytesScanned = byteCounter.getBytesScanned(context);
            // Assertion of claim (c)
            assertThat(bytesScanned, greaterThanOrEqualTo(byteCountsByRecord.get(i)));
            if (i < byteCountsByRecord.size() - 1) {
                // Assertion of claim (c)
                assertThat(bytesScanned, lessThanOrEqualTo(byteCountsByRecord.get(i) + byteCountsByRecord.get(i + 1)));
            }
            i++;
            if (result.hasNext()) {
                result = cursor.getNext();
                assertFalse(result.hasNext());
            }
            context.getTimer().reset();
        }
    }
}

Example 19 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method queryWithWideOrOfFullTextPrefixPredicates.

/**
 * Queries with an OR of {@link com.apple.foundationdb.record.query.expressions.Text#containsPrefix(String)}
 * predicates get planned as {@link com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan}s,
 * which have unusual semantics where results are returned in an undefined order as soon as any child has one.
 * Therefore, the assertions made in {@link #assertPlanLimitsWithCorrectExecution(List, FDBRecordContext, RecordQueryPlan)}
 * are far too strong for plans like this. Instead, we make very weak assertions that the byte scan limit does
 * <em>something</em>.
 */
@ParameterizedTest
@MethodSource("complexTextQueries")
public void queryWithWideOrOfFullTextPrefixPredicates(@Nonnull RecordQuery query, int numPredicates) throws Exception {
    deleteSimpleRecords();
    final List<String> textSamples = ImmutableList.of(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH, TextSamples.KOREAN);
    RecordMetaDataHook indexHook = metaDataBuilder -> metaDataBuilder.addIndex(metaDataBuilder.getRecordType(SIMPLE_DOC), SIMPLE_TEXT_PREFIX);
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        for (int i = 0; i < textSamples.size(); i++) {
            recordStore.saveRecord(TestRecordsTextProto.SimpleDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).build());
        }
        commit(context);
    }
    setupPlanner(null);
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, descendant(unorderedUnion(Collections.nCopies(numPredicates, any(RecordQueryPlan.class)))));
    long totalBytes;
    Set<Long> noLimitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        context.getTimer().reset();
        RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, null, ExecuteProperties.SERIAL_EXECUTE);
        RecordCursorResult<FDBQueriedRecord<Message>> result;
        do {
            result = cursor.onNext().get();
            if (result.hasNext()) {
                TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                record.mergeFrom(result.get().getRecord());
                noLimitRecordIds.add(record.getDocId());
            }
        } while (result.hasNext());
        totalBytes = byteCounter.getBytesScanned(context);
    }
    Set<Long> limitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context);
        ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(0);
        byte[] continuation = null;
        do {
            context.getTimer().reset();
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, continuation, executeProperties.build());
            RecordCursorResult<FDBQueriedRecord<Message>> result;
            do {
                result = cursor.onNext().get();
                if (result.hasNext()) {
                    TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                    record.mergeFrom(result.get().getRecord());
                    limitRecordIds.add(record.getDocId());
                }
            } while (result.hasNext());
            assertThat(byteCounter.getBytesScanned(context), lessThan(totalBytes));
            continuation = result.getContinuation().toBytes();
            if (continuation != null) {
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
        } while (continuation != null);
        assertEquals(noLimitRecordIds, limitRecordIds);
    }
}

Example 20 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method testWithFailOnByteScanLimitReached.

@ParameterizedTest(name = "testWithFailOnByteScanLimitReached [{index}] {0} {1}")
@MethodSource("plans")
public void testWithFailOnByteScanLimitReached(String description, boolean notUsed, @Nonnull RecordQueryPlan plan) throws Exception {
    setupSimpleRecordStore();
    for (long byteLimit = 0; byteLimit < 1000; byteLimit += 100) {
        try (FDBRecordContext context = openContext()) {
            ExecuteProperties properties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteLimit).setFailOnScanLimitReached(true).build();
            openSimpleRecordStore(context);
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, properties);
            assertThrowsWithWrapper(ScanLimitReachedException.class, () -> cursor.asList().join());
        }
    }
}