Examples with RecordCursor com.apple.foundationdb.record.RecordCursor used on opensource projects

Example 21 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class SortCursorTests method memorySortContinuations.

@Test
public void memorySortContinuations() throws Exception {
    final Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> scanRecords = continuation -> {
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(20).build();
        return recordStore.scanRecords(null, null, EndpointType.TREE_START, EndpointType.TREE_END, continuation, new ScanProperties(executeProperties)).map(FDBQueriedRecord::stored);
    };
    final MemoryAdapterBase adapter = new MemoryAdapterBase() {

        @Override
        public int getMaxRecordCountInMemory() {
            return 10;
        }
    };
    List<Integer> resultNums = new ArrayList<>();
    byte[] continuation = null;
    int transactionCount = 0;
    do {
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context);
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = MemorySortCursor.create(adapter, scanRecords, timer, continuation)) {
                while (true) {
                    RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
                    if (result.hasNext()) {
                        int num2 = TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(result.get().getRecord()).getNumValue2();
                        resultNums.add(num2);
                    } else {
                        continuation = result.getContinuation().toBytes();
                        break;
                    }
                }
            }
            transactionCount++;
        }
    } while (continuation != null);
    assertEquals(110, transactionCount);
    assertEquals(sortedNums, resultNums);
}

Example 22 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class SortCursorTests method memorySort.

@Test
public void memorySort() throws Exception {
    final Function<byte[], RecordCursor<FDBQueriedRecord<Message>>> scanRecords = continuation -> recordStore.scanRecords(null, null, EndpointType.TREE_START, EndpointType.TREE_END, continuation, ScanProperties.FORWARD_SCAN).map(FDBQueriedRecord::stored);
    final MemoryAdapterBase adapter = new MemoryAdapterBase() {

        @Override
        public int getMaxRecordCountInMemory() {
            return 20;
        }
    };
    List<Integer> resultNums;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = MemorySortCursor.create(adapter, scanRecords, timer, null)) {
            resultNums = cursor.map(r -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(r.getRecord()).getNumValue2()).asList().get();
        }
    }
    assertEquals(sortedNums.subList(0, 20), resultNums);
}

Example 23 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreQueryTestBase method querySimpleRecordStoreWithContinuation.

/**
 * A query execution utility that can handle continuations. This is very similar to the above {@link #querySimpleRecordStore}
 * with the additional support for {@link ExecuteProperties} and continuation.
 * This method returns the last result encountered. In the case where the row limit was encountered, this would be the one
 * result that contains the continuation that should be used on the next call.
 * @param recordMetaDataHook Metadata hook to invoke while opening store
 * @param plan the plan to execute
 * @param contextSupplier provider method to get execution context
 * @param continuation execution continuation
 * @param executeProperties execution properties to pass into the execute method
 * @param checkNumRecords Consumer that verifies correct number of records returned
 * @param checkRecord Consumer that asserts every record retrieved
 * @param checkDiscarded Consumer that asserts the number of discarded records
 * @return the last result from the cursor
 * @throws Throwable any thrown exception, or its cause if the exception is a {@link ExecutionException}
 */
protected RecordCursorResult<FDBQueriedRecord<Message>> querySimpleRecordStoreWithContinuation(@Nonnull RecordMetaDataHook recordMetaDataHook, @Nonnull RecordQueryPlan plan, @Nonnull Supplier<EvaluationContext> contextSupplier, @Nullable byte[] continuation, @Nonnull ExecuteProperties executeProperties, @Nonnull Consumer<Integer> checkNumRecords, @Nonnull Consumer<TestRecords1Proto.MySimpleRecord.Builder> checkRecord, @Nonnull Consumer<FDBRecordContext> checkDiscarded) throws Throwable {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, recordMetaDataHook);
        AtomicInteger i = new AtomicInteger(0);
        CompletableFuture<RecordCursorResult<FDBQueriedRecord<Message>>> lastResult;
        RecordCursor<FDBQueriedRecord<Message>> cursor = plan.execute(recordStore, contextSupplier.get(), continuation, executeProperties);
        lastResult = cursor.forEachResult(result -> {
            TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
            myrec.mergeFrom(result.get().getRecord());
            checkRecord.accept(myrec);
            i.incrementAndGet();
        });
        lastResult.get();
        checkNumRecords.accept(i.get());
        checkDiscarded.accept(context);
        // TODO a hack until this gets refactored properly
        clearStoreCounter(context);
        return lastResult.get();
    } catch (ExecutionException ex) {
        throw ex.getCause();
    }
}

Example 24 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class FDBRestrictedIndexQueryTest method queryAllowedIndexes.

/**
 * Verify that queries do not use prohibited indexes.
 */
@DualPlannerTest
void queryAllowedIndexes() {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$str_value_indexed");
        metaData.addIndex("MySimpleRecord", new Index("limited_str_value_index", field("str_value_indexed"), Index.EMPTY_VALUE, IndexTypes.VALUE, IndexOptions.NOT_ALLOWED_FOR_QUERY_OPTIONS));
    };
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteAllRecords();
        TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(1);
        recBuilder.setStrValueIndexed("abc");
        recBuilder.setNumValueUnique(123);
        recordStore.saveRecord(recBuilder.build());
        recBuilder.setRecNo(2);
        recBuilder.setStrValueIndexed("xyz");
        recBuilder.setNumValueUnique(987);
        recordStore.saveRecord(recBuilder.build());
        commit(context);
    }
    RecordQuery query1 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("abc")).build();
    // Index(limited_str_value_index [[abc],[abc]])
    // Scan(<,>) | [MySimpleRecord] | str_value_indexed EQUALS abc
    RecordQueryPlan plan1 = planner.plan(query1);
    assertThat("should not use prohibited index", plan1, hasNoDescendant(indexScan("limited_str_value_index")));
    assertTrue(plan1.hasFullRecordScan(), "should use full record scan");
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(-223683738, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
    // TODO: Issue https://github.com/FoundationDB/fdb-record-layer/issues/1074
    // assertEquals(1148834070, plan1.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(-2136471589, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan1)) {
            FDBQueriedRecord<Message> rec = cursor.getNext().get();
            TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
            myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
            assertEquals("abc", myrec.getStrValueIndexed());
            assertFalse(cursor.getNext().hasNext());
        }
        TestHelpers.assertDiscardedExactly(1, context);
        clearStoreCounter(context);
    }
    RecordQuery query2 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("abc")).setAllowedIndex("limited_str_value_index").build();
    // Index(limited_str_value_index [[abc],[abc]])
    RecordQueryPlan plan2 = planner.plan(query2);
    assertThat("explicitly use prohibited index", plan2, descendant(indexScan("limited_str_value_index")));
    assertFalse(plan2.hasRecordScan(), "should not use record scan");
    assertEquals(-1573180774, plan2.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(994464666, plan2.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1531627068, plan2.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan2)) {
            FDBQueriedRecord<Message> rec = cursor.getNext().get();
            TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
            myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
            assertEquals("abc", myrec.getStrValueIndexed());
            assertFalse(cursor.getNext().hasNext());
        }
        TestHelpers.assertDiscardedNone(context);
    }
}

Example 25 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectoryTest method testListObeysTimeLimits.

@Test
public void testListObeysTimeLimits() {
    KeySpace root = new KeySpace(new KeySpaceDirectory("root", KeyType.STRING, "root-" + random.nextInt(Integer.MAX_VALUE)).addSubdirectory(new KeySpaceDirectory("a", KeyType.LONG).addSubdirectory(new KeySpaceDirectory("b", KeyType.LONG).addSubdirectory(new KeySpaceDirectory("c", KeyType.LONG)))));
    final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    try (FDBRecordContext context = database.openContext()) {
        Transaction tr = context.ensureActive();
        for (int i = 0; i < 10; i++) {
            for (int j = 0; j < 3; j++) {
                for (int k = 0; k < 5; k++) {
                    tr.set(root.path("root").add("a", i).add("b", j).add("c", k).toTuple(context).pack(), Tuple.from(i + j).pack());
                }
            }
        }
        tr.commit().join();
    }
    try (FDBRecordContext context = database.openContext()) {
        // Iteration will inject a 1ms pause in each "a" value we iterate over (there are 10 of them)
        // so we want to make the time limit long enough to make *some* progress, but short enough to
        // to make sure we cannot get them all.
        ScanProperties props = new ScanProperties(ExecuteProperties.newBuilder().setFailOnScanLimitReached(false).setTimeLimit(5L).build());
        // The inner and outer iterator are declared here instead of in-line with the call to flatMapPipelined
        // because IntelliJ was having issues groking the call as a single call.
        Function<byte[], RecordCursor<ResolvedKeySpacePath>> aIterator = outerContinuation -> root.path("root").listSubdirectoryAsync(context, "a", outerContinuation, props).map(value -> {
            sleep(1L);
            return value;
        });
        BiFunction<ResolvedKeySpacePath, byte[], RecordCursor<ResolvedKeySpacePath>> bIterator = (aPath, innerContinuation) -> aPath.toPath().add("b", 0).listSubdirectoryAsync(context, "c", innerContinuation, props);
        RecordCursor<ResolvedKeySpacePath> cursor = RecordCursor.flatMapPipelined(aIterator, bIterator, null, 10);
        long count = cursor.getCount().join();
        assertEquals(RecordCursor.NoNextReason.TIME_LIMIT_REACHED, cursor.getNext().getNoNextReason());
        // With a 1ms delay we should read no more than 5 "a" values (there are a total of 10)
        // and each "c" value has 4 values. so we shouldn't have been able to read more than 40
        // total values.
        assertTrue(count <= 40, "Read too many values, query should have timed out");
    }
}