Examples with ExecuteProperties com.apple.foundationdb.record.ExecuteProperties used on opensource projects

Example 11 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class SizeStatisticsCollectorTest method records100.

@Test
public void records100() throws Exception {
    final int recordCount = 100;
    final int keyBytes;
    final int valueBytes;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int i = 0; i < recordCount; i++) {
            MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().setRecNo(i).setStrValueIndexed(i % 2 == 0 ? "even" : "odd").build();
            recordStore.saveRecord(simpleRecord);
        }
        keyBytes = recordStore.getTimer().getCount(FDBStoreTimer.Counts.SAVE_RECORD_KEY_BYTES);
        valueBytes = recordStore.getTimer().getCount(FDBStoreTimer.Counts.SAVE_RECORD_VALUE_BYTES);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        SizeStatisticsCollector statisticsCollector = new SizeStatisticsCollector(recordStore);
        assertThat(statisticsCollector.collect(context, ExecuteProperties.SERIAL_EXECUTE), is(true));
        assertEquals(recordCount * 2, statisticsCollector.getKeyCount());
        assertEquals(keyBytes, statisticsCollector.getKeySize());
        assertEquals(valueBytes, statisticsCollector.getValueSize());
        assertEquals(keyBytes + valueBytes, statisticsCollector.getTotalSize());
        assertEquals(keyBytes * 0.5 / recordCount, statisticsCollector.getAverageKeySize());
        assertEquals(valueBytes * 0.5 / recordCount, statisticsCollector.getAverageValueSize());
        // Batches of 10
        SizeStatisticsCollector batchedCollector = new SizeStatisticsCollector(recordStore);
        ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(10).build();
        boolean done = false;
        int iterations = 0;
        while (!done) {
            done = batchedCollector.collect(context, executeProperties);
            iterations += 1;
        }
        assertThat(iterations, anyOf(equalTo(recordCount * 2 / 10), equalTo(recordCount * 2 / 10 + 1)));
        assertEquals(statisticsCollector.getKeyCount(), batchedCollector.getKeyCount());
        assertEquals(statisticsCollector.getKeySize(), batchedCollector.getKeySize());
        assertEquals(statisticsCollector.getValueSize(), batchedCollector.getValueSize());
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        SizeStatisticsCollector indexCollector = new SizeStatisticsCollector(recordStore, "MySimpleRecord$str_value_indexed");
        // Batches of 10
        ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(10).build();
        boolean done = false;
        int iterations = 0;
        while (!done) {
            done = indexCollector.collect(context, executeProperties);
            iterations += 1;
        }
        assertThat(iterations, anyOf(equalTo(recordCount / 10), equalTo(recordCount / 10 + 1)));
        assertEquals(recordCount, indexCollector.getKeyCount());
        assertEquals(0, indexCollector.getValueSize());
        commit(context);
    }
}

Example 12 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class SplitHelperTest method scanContinuations.

@MethodSource("limitsAndReverseArgs")
@ParameterizedTest(name = "scanContinuations [returnLimit = {0}, readLimit = {1}, reverse = {2}]")
public void scanContinuations(final int returnLimit, final int readLimit, final boolean reverse) throws Exception {
    List<FDBRawRecord> rawRecords = writeDummyRecords();
    if (reverse) {
        rawRecords = Lists.reverse(rawRecords);
    }
    final Iterator<FDBRawRecord> expectedRecordIterator = rawRecords.iterator();
    try (FDBRecordContext context = openContext()) {
        byte[] continuation = null;
        do {
            final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(returnLimit).setScannedRecordsLimit(readLimit).build();
            ScanProperties scanProperties = new ScanProperties(executeProperties, reverse);
            RecordCursor<KeyValue> kvCursor = KeyValueCursor.Builder.withSubspace(subspace).setContext(context).setRange(TupleRange.ALL).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).setContinuation(continuation).build();
            RecordCursorIterator<FDBRawRecord> recordCursor = new SplitHelper.KeyValueUnsplitter(context, subspace, kvCursor, false, null, scanProperties.with(ExecuteProperties::clearReturnedRowLimit)).limitRowsTo(returnLimit).asIterator();
            int retrieved = 0;
            int rowsScanned = 0;
            while (recordCursor.hasNext()) {
                assertThat(retrieved, lessThan(returnLimit));
                assertThat(rowsScanned, lessThanOrEqualTo(readLimit));
                FDBRawRecord nextRecord = recordCursor.next();
                assertNotNull(nextRecord);
                assertThat(expectedRecordIterator.hasNext(), is(true));
                FDBRawRecord expectedRecord = expectedRecordIterator.next();
                assertEquals(expectedRecord, nextRecord);
                rowsScanned += nextRecord.getKeyCount();
                retrieved += 1;
            }
            if (retrieved > 0) {
                continuation = recordCursor.getContinuation();
                if (retrieved >= returnLimit) {
                    assertEquals(RecordCursor.NoNextReason.RETURN_LIMIT_REACHED, recordCursor.getNoNextReason());
                    assertNotNull(continuation);
                } else if (rowsScanned > readLimit) {
                    assertEquals(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, recordCursor.getNoNextReason());
                    assertNotNull(continuation);
                } else if (rowsScanned < readLimit) {
                    assertEquals(RecordCursor.NoNextReason.SOURCE_EXHAUSTED, recordCursor.getNoNextReason());
                } else {
                    // If we read exactly as many records as is allowed by the read record limit, then
                    // this probably means that we hit SCAN_LIMIT_REACHED, but it's also possible to
                    // hit SOURCE_EXHAUSTED if we hit the record read limit at exactly the same time
                    // as we needed to do another speculative read to determine if a split record
                    // continues or not.
                    assertEquals(readLimit, rowsScanned);
                    assertThat(recordCursor.getNoNextReason(), is(oneOf(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, RecordCursor.NoNextReason.SOURCE_EXHAUSTED)));
                    if (!recordCursor.getNoNextReason().isSourceExhausted()) {
                        assertNotNull(recordCursor.getContinuation());
                    }
                }
            } else {
                assertNull(recordCursor.getContinuation());
                continuation = null;
            }
        } while (continuation != null);
        commit(context);
    }
}

Example 13 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class SortCursorTests method memorySortContinuations.

@Test
public void memorySortContinuations() throws Exception {
    final Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> scanRecords = continuation -> {
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(20).build();
        return recordStore.scanRecords(null, null, EndpointType.TREE_START, EndpointType.TREE_END, continuation, new ScanProperties(executeProperties)).map(FDBQueriedRecord::stored);
    };
    final MemoryAdapterBase adapter = new MemoryAdapterBase() {

        @Override
        public int getMaxRecordCountInMemory() {
            return 10;
        }
    };
    List<Integer> resultNums = new ArrayList<>();
    byte[] continuation = null;
    int transactionCount = 0;
    do {
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context);
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = MemorySortCursor.create(adapter, scanRecords, timer, continuation)) {
                while (true) {
                    RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
                    if (result.hasNext()) {
                        int num2 = TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(result.get().getRecord()).getNumValue2();
                        resultNums.add(num2);
                    } else {
                        continuation = result.getContinuation().toBytes();
                        break;
                    }
                }
            }
            transactionCount++;
        }
    } while (continuation != null);
    assertEquals(110, transactionCount);
    assertEquals(sortedNums, resultNums);
}

Example 14 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreQueryTestBase method querySimpleRecordStoreWithContinuation.

/**
 * A query execution utility that can handle continuations. This is very similar to the above {@link #querySimpleRecordStore}
 * with the additional support for {@link ExecuteProperties} and continuation.
 * This method returns the last result encountered. In the case where the row limit was encountered, this would be the one
 * result that contains the continuation that should be used on the next call.
 * @param recordMetaDataHook Metadata hook to invoke while opening store
 * @param plan the plan to execute
 * @param contextSupplier provider method to get execution context
 * @param continuation execution continuation
 * @param executeProperties execution properties to pass into the execute method
 * @param checkNumRecords Consumer that verifies correct number of records returned
 * @param checkRecord Consumer that asserts every record retrieved
 * @param checkDiscarded Consumer that asserts the number of discarded records
 * @return the last result from the cursor
 * @throws Throwable any thrown exception, or its cause if the exception is a {@link ExecutionException}
 */
protected RecordCursorResult<FDBQueriedRecord<Message>> querySimpleRecordStoreWithContinuation(@Nonnull RecordMetaDataHook recordMetaDataHook, @Nonnull RecordQueryPlan plan, @Nonnull Supplier<EvaluationContext> contextSupplier, @Nullable byte[] continuation, @Nonnull ExecuteProperties executeProperties, @Nonnull Consumer<Integer> checkNumRecords, @Nonnull Consumer<TestRecords1Proto.MySimpleRecord.Builder> checkRecord, @Nonnull Consumer<FDBRecordContext> checkDiscarded) throws Throwable {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, recordMetaDataHook);
        AtomicInteger i = new AtomicInteger(0);
        CompletableFuture<RecordCursorResult<FDBQueriedRecord<Message>>> lastResult;
        RecordCursor<FDBQueriedRecord<Message>> cursor = plan.execute(recordStore, contextSupplier.get(), continuation, executeProperties);
        lastResult = cursor.forEachResult(result -> {
            TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
            myrec.mergeFrom(result.get().getRecord());
            checkRecord.accept(myrec);
            i.incrementAndGet();
        });
        lastResult.get();
        checkNumRecords.accept(i.get());
        checkDiscarded.accept(context);
        // TODO a hack until this gets refactored properly
        clearStoreCounter(context);
        return lastResult.get();
    } catch (ExecutionException ex) {
        throw ex.getCause();
    }
}

Example 15 with ExecuteProperties

use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreScanLimitTest method unorderedIntersectionWithScanLimit.

@ParameterizedTest(name = "unorderedIntersectionWithScanLimit [fail = {0}]")
@BooleanSource
public void unorderedIntersectionWithScanLimit(boolean fail) throws Exception {
    // TODO: When there is an UnorderedIntersectionPlan (or whatever) add that to the unordered plans stream
    RecordQueryPlanner planner = new RecordQueryPlanner(simpleMetaData(NO_HOOK), new RecordStoreState(null, null));
    RecordQueryPlan leftPlan = planner.plan(RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").startsWith("ev")).build());
    RecordQueryPlan rightPlan = planner.plan(RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").lessThanOrEquals(1)).build());
    int maximumToScan;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        maximumToScan = recordStore.executeQuery(leftPlan).getCount().get() + recordStore.executeQuery(rightPlan).getCount().get();
    }
    for (int limit = 0; limit < 3 * maximumToScan; limit = 2 * limit + 1) {
        final int finalLimit = limit;
        Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> cursorFunction = (continuation) -> {
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(finalLimit).setFailOnScanLimitReached(fail).build();
            return ProbableIntersectionCursor.create(record -> record.getPrimaryKey().getItems(), Arrays.asList(leftContinuation -> leftPlan.execute(recordStore, EvaluationContext.EMPTY, leftContinuation, executeProperties), rightContinuation -> rightPlan.execute(recordStore, EvaluationContext.EMPTY, rightContinuation, executeProperties)), continuation, recordStore.getTimer());
        };
        assertNumberOfRecordsScanned(limit, cursorFunction, fail, "should" + (limit >= maximumToScan ? "not " : "") + " be limited by record scan limit");
    }
}