Examples with RecordCursor com.apple.foundationdb.record.RecordCursor used on opensource projects

Example 16 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method andOrQueryWithContinuation.

@Test
void andOrQueryWithContinuation() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        saveRecords(100, 200);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.field("num_value_2").equalsValue(3), Query.or(Query.field("num_value_3").equalsValue(1), Query.field("num_value_3").equalsValue(4))));
        List<Integer> onBits = new ArrayList<>();
        int ntimes = 0;
        byte[] continuation = null;
        do {
            RecordCursor<IndexEntry> cursor = queryPlan.execute(recordStore, EvaluationContext.EMPTY, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).map(FDBQueriedRecord::getIndexEntry);
            RecordCursorResult<IndexEntry> cursorResult = cursor.forEachResult(i -> onBits.addAll(collectOnBits(i.get()))).join();
            ntimes++;
            continuation = cursorResult.getContinuation().toBytes();
        } while (continuation != null);
        assertThat(onBits, equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && ((i % 5) == 1 || (i % 5) == 4)).collect(Collectors.toList())));
        assertThat(ntimes, equalTo(4));
    }
}

Example 17 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class UnorderedUnionCursorTest method loopIterationWithLimit.

@Test
public void loopIterationWithLimit() throws ExecutionException, InterruptedException {
    FDBStoreTimer timer = new FDBStoreTimer();
    FirableCursor<Integer> secondCursor = new FirableCursor<>(RecordCursor.fromList(Arrays.asList(3, 4)));
    RecordCursor<Integer> cursor = UnorderedUnionCursor.create(Arrays.asList(continuation -> RecordCursor.fromList(Arrays.asList(1, 2), continuation).limitRowsTo(1), continuation -> secondCursor), null, timer);
    RecordCursorResult<Integer> cursorResult = cursor.getNext();
    assertEquals(1, (int) cursorResult.get());
    CompletableFuture<RecordCursorResult<Integer>> cursorResultFuture = cursor.onNext();
    assertFalse(cursorResultFuture.isDone());
    secondCursor.fire();
    cursorResult = cursorResultFuture.get();
    assertEquals(3, (int) cursorResult.get());
    cursorResultFuture = cursor.onNext();
    assertFalse(cursorResultFuture.isDone());
    secondCursor.fire();
    cursorResult = cursorResultFuture.get();
    assertEquals(4, (int) cursorResult.get());
    cursorResultFuture = cursor.onNext();
    assertFalse(cursorResultFuture.isDone());
    secondCursor.fire();
    cursorResult = cursorResultFuture.get();
    assertFalse(cursorResult.hasNext());
    assertEquals(RecordCursor.NoNextReason.RETURN_LIMIT_REACHED, cursorResult.getNoNextReason());
    assertThat(timer.getCount(FDBStoreTimer.Events.QUERY_INTERSECTION), lessThanOrEqualTo(5));
}

Example 18 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class GeophileQueryTest method testDistance.

@Test
@Tag(Tags.Slow)
public void testDistance() throws Exception {
    final RecordMetaDataHook hook = md -> {
        md.addIndex("City", CITY_LOCATION_COVERING_INDEX);
    };
    loadCities(hook, 0);
    final int centerId = 5391959;
    final double distance = 1;
    final int scanLimit = 5000;
    final RecordQueryPlan scanPlan = distanceFilterScan(distance);
    final Set<Integer> scanResults = new HashSet<>();
    byte[] continuation = null;
    do {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context, hook);
            EvaluationContext joinContext = bindCenter(centerId);
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build();
            RecordCursor<FDBQueriedRecord<Message>> recordCursor = scanPlan.execute(recordStore, joinContext, continuation, executeProperties);
            recordCursor.forEach(city -> {
                TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
                LOGGER.debug(String.format("Scan found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
                scanResults.add(cityBuilder.getGeoNameId());
            }).join();
            continuation = recordCursor.getNext().getContinuation().toBytes();
            commit(context);
        }
    } while (continuation != null);
    final RecordQueryPlan indexPlan = distanceSpatialQuery(distance, false);
    final Set<Integer> indexResults = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<FDBQueriedRecord<Message>> recordCursor = indexPlan.execute(recordStore, bindCenter(centerId));
        recordCursor.forEach(city -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
            LOGGER.debug(String.format("Index found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
            indexResults.add(cityBuilder.getGeoNameId());
        }).join();
        int given = timer.getCount(FDBStoreTimer.Counts.QUERY_FILTER_GIVEN);
        int passed = timer.getCount(FDBStoreTimer.Counts.QUERY_FILTER_PASSED);
        int discarded = timer.getCount(FDBStoreTimer.Counts.QUERY_DISCARDED);
        assertThat("Should have passed more than discarded", passed, greaterThan(discarded));
        commit(context);
    }
    final RecordQueryPlan coveringPlan = distanceSpatialQuery(distance, true);
    final Set<Integer> coveringResults = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<FDBQueriedRecord<Message>> recordCursor = indexPlan.execute(recordStore, bindCenter(centerId));
        recordCursor.forEach(city -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
            LOGGER.debug(String.format("Covering found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
            coveringResults.add(cityBuilder.getGeoNameId());
        }).join();
        commit(context);
    }
    assertEquals(scanResults, indexResults);
    assertEquals(scanResults, coveringResults);
}

Example 19 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class QueryPlanCursorTest method compareSkipsAndCursors.

private void compareSkipsAndCursors(RecordQueryPlan plan, int amount) throws Exception {
    final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
        TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
        record.mergeFrom(r.getRecord());
        return record.getRecNo();
    };
    final ExecuteProperties justLimit = ExecuteProperties.newBuilder().setReturnedRowLimit(amount).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final List<Long> whole;
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            whole = cursor.map(getRecNo).asList().get();
        }
        assertTrue(whole.size() > amount, "should have more than one batch");
        final List<Long> byCursors = new ArrayList<>();
        byte[] continuation = null;
        do {
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, justLimit)) {
                RecordCursorResult<FDBQueriedRecord<Message>> result = null;
                do {
                    result = cursor.getNext();
                    if (result.hasNext()) {
                        byCursors.add(getRecNo.apply(result.get()));
                    }
                } while (result.hasNext());
                continuation = result.getContinuation().toBytes();
            }
        } while (continuation != null);
        assertEquals(whole, byCursors);
        final List<Long> byOffsets = new ArrayList<>();
        int offset = 0;
        while (true) {
            final ExecuteProperties skipAndLimit = ExecuteProperties.newBuilder().setSkip(offset).setReturnedRowLimit(amount).setIsolationLevel(justLimit.getIsolationLevel()).build();
            try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, skipAndLimit)) {
                final List<Long> next = cursor.map(getRecNo).asList().get();
                byOffsets.addAll(next);
                if (next.size() < amount) {
                    break;
                } else {
                    offset += next.size();
                }
            }
        }
        assertEquals(whole, byOffsets);
    }
}

Example 20 with RecordCursor

use of com.apple.foundationdb.record.RecordCursor in project fdb-record-layer by FoundationDB.

the class ProbableIntersectionCursorTest method resumeFromContinuation.

/**
 * Test that the cursor can be resumed by deserializing its state from the continuation object.
 */
@Test
public void resumeFromContinuation() {
    final FDBStoreTimer timer = new FDBStoreTimer();
    final List<Integer> list1 = Arrays.asList(10, 2, 5, 6, 8, 19, 0);
    final List<Integer> list2 = Arrays.asList(9, 1, 3, 5, 2, 4, 8);
    final List<Function<byte[], RecordCursor<Integer>>> cursorFuncs = listsToFunctions(Arrays.asList(list1, list2));
    final Function<byte[], ProbableIntersectionCursor<Integer>> intersectionCursorFunction = continuation -> ProbableIntersectionCursor.create(Collections::singletonList, cursorFuncs, continuation, timer);
    final Iterator<Integer> resultIterator = Iterators.forArray(5, 2, 8);
    byte[] continuation = null;
    boolean done = false;
    List<BloomFilter<List<Object>>> lastBloomFilters = null;
    while (!done) {
        ProbableIntersectionCursor<Integer> intersectionCursor = intersectionCursorFunction.apply(continuation);
        List<BloomFilter<List<Object>>> bloomFilters = intersectionCursor.getCursorStates().stream().map(ProbableIntersectionCursorState::getBloomFilter).collect(Collectors.toList());
        if (lastBloomFilters != null) {
            assertEquals(lastBloomFilters, bloomFilters);
        }
        lastBloomFilters = bloomFilters;
        RecordCursorResult<Integer> result = intersectionCursor.getNext();
        if (resultIterator.hasNext()) {
            assertThat(result.hasNext(), is(true));
            assertEquals(resultIterator.next(), result.get());
            assertThat(result.getContinuation().isEnd(), is(false));
            assertNotNull(result.getContinuation().toBytes());
        } else {
            assertThat(result.hasNext(), is(false));
            assertEquals(RecordCursor.NoNextReason.SOURCE_EXHAUSTED, result.getNoNextReason());
            assertThat(result.getContinuation().isEnd(), is(true));
            assertNull(result.getContinuation().toBytes());
            done = true;
        }
        continuation = result.getContinuation().toBytes();
    }
    assertEquals(3, timer.getCount(FDBStoreTimer.Counts.QUERY_INTERSECTION_PLAN_MATCHES));
    assertEquals(list1.size() + list2.size() - 3, timer.getCount(FDBStoreTimer.Counts.QUERY_INTERSECTION_PLAN_NONMATCHES));
}