Examples with RecordCursorIterator com.apple.foundationdb.record.RecordCursorIterator used on opensource projects

Example 16 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class ChainedCursorTest method testObeysTimeLimit.

@Test
public void testObeysTimeLimit() {
    FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    try (FDBRecordContext context = database.openContext()) {
        ScanProperties props = new ScanProperties(ExecuteProperties.newBuilder().setTimeLimit(4L).setFailOnScanLimitReached(false).build());
        RecordCursorIterator<Long> cursor = new ChainedCursor<>(context, (lastKey) -> nextKey(lastKey).thenApply(value -> {
            sleep(1L);
            return value;
        }), (key) -> Tuple.from(key).pack(), (prevContinuation) -> Tuple.fromBytes(prevContinuation).getLong(0), null, props).asIterator();
        int count = 0;
        while (cursor.hasNext()) {
            assertEquals(Long.valueOf(count), cursor.next());
            ++count;
        }
        assertEquals(cursor.getNoNextReason(), RecordCursor.NoNextReason.TIME_LIMIT_REACHED);
        assertTrue(count < 5, "Too many values returned");
    }
}

Example 17 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class TextIndexTest method printUsage.

private void printUsage() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        Subspace indexSubspace = recordStore.getIndexMaintainer(recordStore.getRecordMetaData().getIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME)).getIndexSubspace();
        final int indexSuspaceLength = indexSubspace.getKey().length;
        int subspaceOverhead = 0;
        int keySize = 0;
        int valueSize = 0;
        for (KeyValue kv : context.ensureActive().getRange(indexSubspace.range())) {
            subspaceOverhead += indexSuspaceLength;
            keySize += kv.getKey().length - indexSuspaceLength;
            valueSize += kv.getValue().length;
        }
        int textSize = 0;
        RecordCursorIterator<String> cursor = recordStore.scanRecords(null, ScanProperties.FORWARD_SCAN).map(record -> {
            Message msg = record.getRecord();
            Descriptors.FieldDescriptor fd = msg.getDescriptorForType().findFieldByName("text");
            return msg.getField(fd).toString();
        }).asIterator();
        while (cursor.hasNext()) {
            textSize += cursor.next().length();
        }
        LOGGER.info("Usage:");
        LOGGER.info("  Subspace: {} kB", subspaceOverhead * 1e-3);
        LOGGER.info("  Keys:     {} kB", keySize * 1e-3);
        LOGGER.info("  Values:   {} kB", valueSize * 1e-3);
        LOGGER.info("  Text:     {} kB", textSize * 1e-3);
        LOGGER.info("  Overhead: {}", textSize == 0.0 ? Double.POSITIVE_INFINITY : ((subspaceOverhead + keySize + valueSize) * 1.0 / textSize));
    }
}

Example 18 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method queryOnRepeatedVersion.

@ParameterizedTest(name = "queryOnRepeatedVersions [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnRepeatedVersion(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> {
        MySimpleRecord.Builder builder = MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3);
        for (int i = 0; i < id % 3; i++) {
            builder.addRepeater(i);
        }
        return builder.build();
    }).collect(Collectors.toList());
    Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
    while (simpleIterator.hasNext()) {
        try (FDBRecordContext context = openContext(repeatedVersionHook)) {
            int done = 0;
            while (simpleIterator.hasNext() && done != 5) {
                recordStore.saveRecord(simpleIterator.next());
                done += 1;
            }
            context.commit();
        }
    }
    try (FDBRecordContext context = openContext(repeatedVersionHook)) {
        List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getRepeaterList().contains(1)).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
        FDBRecordVersion last = null;
        List<Long> receivedKeys = new ArrayList<>();
        int totalSeen = 0;
        while (true) {
            RecordQueryPlan plan;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("repeater").oneOfThem().equalsValue(1)).setSort(VersionKeyExpression.VERSION).setRemoveDuplicates(false).build();
                plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$repeater-version [[1],[1]])", plan.toString());
            } else {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("repeater").oneOfThem().equalsValue(1), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).setRemoveDuplicates(false).build();
                plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$repeater-version ([1, " + last.toVersionstamp() + "],[1]])", plan.toString());
            }
            RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(4).build()).asIterator();
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBQueriedRecord<Message> record = cursor.next();
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
                assertThat(simpleRecord.getRepeaterList(), hasItem(1));
                receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals(simpleRecords.size() / 3, totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
}

Example 19 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method validateUsingNewerVersionFormat.

private <M extends Message> void validateUsingNewerVersionFormat(@Nonnull List<FDBStoredRecord<M>> storedRecords) {
    // Make sure the old keyspace doesn't have anything in it
    final Subspace legacyVersionSubspace = recordStore.getLegacyVersionSubspace();
    KeyValueCursor legacyKvs = KeyValueCursor.Builder.withSubspace(legacyVersionSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build();
    assertEquals(0, (int) legacyKvs.getCount().join());
    // Look for the versions within the primary keyspace
    final Subspace recordsSubspace = recordStore.recordsSubspace();
    RecordCursorIterator<Pair<Tuple, FDBRecordVersion>> versionKeyPairs = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().map(kv -> Pair.of(recordsSubspace.unpack(kv.getKey()), kv.getValue())).filter(tupleBytesPair -> tupleBytesPair.getLeft().getLong(tupleBytesPair.getLeft().size() - 1) == -1).map(tupleBytesPair -> Pair.of(tupleBytesPair.getLeft().popBack(), FDBRecordVersion.fromVersionstamp(Tuple.fromBytes(tupleBytesPair.getRight()).getVersionstamp(0)))).asIterator();
    for (FDBStoredRecord<M> storedRecord : storedRecords) {
        assertTrue(versionKeyPairs.hasNext());
        Pair<Tuple, FDBRecordVersion> versionPair = versionKeyPairs.next();
        assertEquals(storedRecord.getPrimaryKey(), versionPair.getLeft());
        assertEquals(storedRecord.getVersion(), versionPair.getRight());
    }
    assertFalse(versionKeyPairs.hasNext());
}

Example 20 with RecordCursorIterator

use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testUnorderableOrQueryWithAnd.

/**
 * Verify that a complex query with an OR of an AND produces a union plan if the appropriate indexes are defined.
 * Unlike {@link #testOrQuery6()}, the legs of the union are not compatibly ordered, so this will revert to using
 * an unordered union.
 */
@DualPlannerTest
@ParameterizedTest
@BooleanSource
void testUnorderableOrQueryWithAnd(boolean removesDuplicates) throws Exception {
    RecordMetaDataHook hook = metaDataBuilder -> {
        complexQuerySetupHook().apply(metaDataBuilder);
        metaDataBuilder.addIndex("MySimpleRecord", new Index("multi_index_2", "str_value_indexed", "num_value_3_indexed"));
    };
    complexQuerySetup(hook);
    setDeferFetchAfterUnionAndIntersection(true);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.or(Query.field("num_value_2").lessThanOrEquals(1), Query.field("num_value_3_indexed").greaterThanOrEquals(3)))).setRemoveDuplicates(removesDuplicates).build();
    // Unordered(Index(multi_index ([even, null],[even, 1]]) ∪ Index(multi_index_2 [[even, 3],[even]]))
    RecordQueryPlan plan = planner.plan(query);
    BindingMatcher<? extends RecordQueryPlan> planMatcher = unorderedUnionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index")).and(scanComparisons(range("([even, null],[even, 1]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index_2")).and(scanComparisons(range("[[even, 3],[even]]"))))));
    if (removesDuplicates) {
        planMatcher = fetchFromPartialRecordPlan(unorderedPrimaryKeyDistinctPlan(planMatcher));
    } else {
        planMatcher = fetchFromPartialRecordPlan(planMatcher);
    }
    assertMatchesExactly(plan, planMatcher);
    assertEquals(removesDuplicates ? -1216499257 : -1216499264, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(removesDuplicates ? 610131412 : 1405311066, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(removesDuplicates ? 1591758672 : -1908028970, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue(myrec.getStrValueIndexed().equals("even") && (myrec.getNumValue2() <= 1 || myrec.getNumValue3Indexed() >= 3));
                i++;
            }
        }
        if (removesDuplicates) {
            assertEquals(40, i);
            assertDiscardedAtMost(13, context);
            assertLoadRecord(53, context);
        } else {
            assertEquals(53, i);
            assertDiscardedNone(context);
        }
    }
}