Examples with RecordQueryPlan com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan used on opensource projects

Example 76 with RecordQueryPlan

use of com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method queryOnVersion.

@ParameterizedTest(name = "queryOnVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnVersion(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
    List<TestRecords1Proto.MyOtherRecord> otherRecords = IntStream.range(0, 30).mapToObj(id -> TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(id * 2 + 1).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
    Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
    Iterator<TestRecords1Proto.MyOtherRecord> otherIterator = otherRecords.iterator();
    while (simpleIterator.hasNext()) {
        try (FDBRecordContext context = openContext(simpleVersionHook)) {
            int done = 0;
            while (simpleIterator.hasNext() && done != 5) {
                recordStore.saveRecord(simpleIterator.next());
                recordStore.saveRecord(otherIterator.next());
                done += 1;
            }
            context.commit();
        }
    }
    // Query all records.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = Stream.concat(simpleRecords.stream().map(MySimpleRecord::getRecNo), otherRecords.stream().map(TestRecords1Proto.MyOtherRecord::getRecNo)).sorted().collect(Collectors.toList());
        FDBRecordVersion last = null;
        List<Long> receivedKeys = new ArrayList<>();
        int totalSeen = 0;
        while (true) {
            RecordQueryPlan plan;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
                plan = planner.plan(query);
                assertEquals("Index(globalVersion <,>)", plan.toString());
            } else {
                RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().greaterThan(last)).setSort(VersionKeyExpression.VERSION).build();
                plan = planner.plan(query);
                assertEquals("Index(globalVersion ([" + last.toVersionstamp() + "],>)", plan.toString());
            }
            RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator();
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBQueriedRecord<Message> record = cursor.next();
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals(simpleRecords.size() + otherRecords.size(), totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query MySimpleRecord based on value.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
        List<Long> receivedKeys = new ArrayList<>();
        FDBRecordVersion last = null;
        int totalSeen = 0;
        while (true) {
            RecordCursorIterator<? extends FDBRecord<Message>> cursor;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version [[0],[0]])", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
            } else {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]])", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
            }
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBRecord<Message> record = cursor.next();
                MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
                assertEquals(0, simpleRecord.getNumValue2());
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(simpleRecord.getRecNo());
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals((simpleRecords.size() + 1) / 2, totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query that requires also filtering
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0 && rec.getNumValue3Indexed() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
        List<Long> receivedKeys = new ArrayList<>();
        FDBRecordVersion last = null;
        int totalSeen = 0;
        while (true) {
            RecordCursorIterator<? extends FDBRecord<Message>> cursor;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version [[0],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
            } else {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
            }
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBRecord<Message> record = cursor.next();
                MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
                assertEquals(0, simpleRecord.getNumValue2());
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(simpleRecord.getRecNo());
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals(simpleRecords.size() / 6, totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query that can't be satisfied with an index scan.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        // Preliminary query to get a read version.
        RecordQuery prelimQuery = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
        RecordQueryPlan prelimPlan = planner.plan(prelimQuery);
        FDBRecordVersion chosenVersion = recordStore.executeQuery(prelimPlan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asList().thenApply(list -> list.get(list.size() - 1).getVersion()).join();
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.version().greaterThan(chosenVersion)).setSort(field("num_value_3_indexed")).build();
        RecordQueryPlan plan = planner.plan(query);
        assertEquals("Index(MySimpleRecord$num_value_3_indexed <,>) | version GREATER_THAN " + chosenVersion.toString(), plan.toString());
        List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
        int last = -1;
        for (FDBQueriedRecord<Message> record : records) {
            MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
            assertThat(last, lessThanOrEqualTo(simpleRecord.getNumValue3Indexed()));
            assertTrue(record.hasVersion());
            assertThat(chosenVersion, lessThan(record.getVersion()));
            last = simpleRecord.getNumValue3Indexed();
        }
        assertEquals(simpleRecords.size() - 5, records.size());
    }
}

Example 77 with RecordQueryPlan

use of com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method withMetaDataRebuilds.

@ParameterizedTest(name = "withMetaDataRebuilds [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void withMetaDataRebuilds(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    RecordMetaDataHook firstHook = metaDataBuilder -> {
        metaDataBuilder.setSplitLongRecords(splitLongRecords);
        metaDataBuilder.addUniversalIndex(new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
        metaDataBuilder.addUniversalIndex(new Index("globalVersion", VersionKeyExpression.VERSION, IndexTypes.VERSION));
    };
    RecordMetaDataHook secondHook = metaDataBuilder -> {
        firstHook.apply(metaDataBuilder);
        metaDataBuilder.removeIndex("globalVersion");
        metaDataBuilder.setStoreRecordVersions(false);
    };
    RecordMetaDataHook thirdHook = metaDataBuilder -> {
        secondHook.apply(metaDataBuilder);
        metaDataBuilder.setStoreRecordVersions(true);
        metaDataBuilder.addUniversalIndex(new Index("globalVersion2", VersionKeyExpression.VERSION, IndexTypes.VERSION));
    };
    MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).build();
    FDBRecordVersion version1;
    MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1776L).build();
    MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(1955L).build();
    FDBRecordVersion version3;
    RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
    // First with version on.
    try (FDBRecordContext context = openContext(firstHook)) {
        FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record1);
        assertTrue(storedRecord.hasVersion());
        context.commit();
        version1 = FDBRecordVersion.complete(context.getVersionStamp(), storedRecord.getVersion().getLocalVersion());
    }
    try (FDBRecordContext context = openContext(firstHook)) {
        FDBStoredRecord<?> loadedRecord = recordStore.loadRecord(Tuple.from(1066L));
        assertNotNull(loadedRecord);
        assertTrue(loadedRecord.hasVersion());
        assertEquals(version1, loadedRecord.getVersion());
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, indexScan("globalVersion"));
        List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
        assertEquals(1, records.size());
        FDBQueriedRecord<Message> queriedRecord = records.get(0);
        assertEquals(Tuple.from(1066L), queriedRecord.getPrimaryKey());
        assertTrue(queriedRecord.hasVersion());
        assertEquals(version1, queriedRecord.getVersion());
    }
    // Now with version off.
    try (FDBRecordContext context = openContext(secondHook)) {
        FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record2);
        assertFalse(storedRecord.hasVersion());
        context.commit();
    }
    try (FDBRecordContext context = openContext(secondHook)) {
        FDBStoredRecord<?> loadedRecord1 = recordStore.loadRecord(Tuple.from(1066L));
        assertNotNull(loadedRecord1);
        assertEquals(testFormatVersion >= FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION, loadedRecord1.hasVersion());
        FDBStoredRecord<?> loadedRecord2 = recordStore.loadRecord(Tuple.from(1776L));
        assertNotNull(loadedRecord2);
        assertFalse(loadedRecord2.hasVersion());
        assertThrows(RecordCoreException.class, () -> {
            RecordQueryPlan plan = planner.plan(query);
            fail("Came up with plan " + plan.toString() + " when it should be impossible");
        });
    }
    // Now with version back on.
    try (FDBRecordContext context = openContext(thirdHook)) {
        FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record3);
        assertTrue(storedRecord.hasVersion());
        context.commit();
        version3 = FDBRecordVersion.complete(context.getVersionStamp(), storedRecord.getVersion().getLocalVersion());
    }
    try (FDBRecordContext context = openContext(thirdHook)) {
        FDBStoredRecord<?> loadedRecord1 = recordStore.loadRecord(Tuple.from(1066L));
        assertEquals(testFormatVersion >= FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION, loadedRecord1.hasVersion());
        FDBStoredRecord<?> loadedRecord2 = recordStore.loadRecord(Tuple.from(1776L));
        assertFalse(loadedRecord2.hasVersion());
        FDBStoredRecord<?> loadedRecord3 = recordStore.loadRecord(Tuple.from(1955L));
        assertTrue(loadedRecord3.hasVersion());
        assertEquals(version3, loadedRecord3.getVersion());
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, indexScan("globalVersion2"));
        List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
        assertEquals(3, records.size());
        if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION) {
            FDBQueriedRecord<Message> queriedRecord1 = records.get(0);
            assertEquals(Tuple.from(1066L), queriedRecord1.getPrimaryKey());
            assertFalse(queriedRecord1.hasVersion());
            FDBQueriedRecord<Message> queriedRecord2 = records.get(1);
            assertEquals(Tuple.from(1776L), queriedRecord2.getPrimaryKey());
            assertFalse(queriedRecord2.hasVersion());
        } else {
            FDBQueriedRecord<Message> queriedRecord1 = records.get(0);
            assertEquals(Tuple.from(1776L), queriedRecord1.getPrimaryKey());
            assertFalse(queriedRecord1.hasVersion());
            FDBQueriedRecord<Message> queriedRecord2 = records.get(1);
            assertEquals(Tuple.from(1066L), queriedRecord2.getPrimaryKey());
            assertTrue(queriedRecord2.hasVersion());
            assertEquals(version1, queriedRecord2.getVersion());
        }
        FDBQueriedRecord<Message> queriedRecord3 = records.get(2);
        assertEquals(Tuple.from(1955L), queriedRecord3.getPrimaryKey());
        assertTrue(queriedRecord3.hasVersion());
        assertEquals(version3, queriedRecord3.getVersion());
    }
}

Example 78 with RecordQueryPlan

use of com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method upgradeFormatVersions.

@ParameterizedTest(name = "upgradeFormatVersions [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void upgradeFormatVersions(int testFormatVersion, boolean splitLongRecords) {
    formatVersion = testFormatVersion;
    final RecordMetaDataHook hook = metaDataBuilder -> {
        simpleVersionHook.apply(metaDataBuilder);
        metaDataBuilder.setSplitLongRecords(splitLongRecords);
    };
    final List<Message> records = Arrays.asList(MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1).build(), MySimpleRecord.newBuilder().setRecNo(1776L).setNumValue2(2).build());
    List<FDBStoredRecord<Message>> storedRecords;
    try (FDBRecordContext context = openContext(hook)) {
        List<FDBStoredRecord<Message>> storedRecordsWithIncompletes = records.stream().map(recordStore::saveRecord).collect(Collectors.toList());
        context.commit();
        byte[] globalVersion = context.getVersionStamp();
        storedRecords = storedRecordsWithIncompletes.stream().map(record -> record.withCommittedVersion(globalVersion)).collect(Collectors.toList());
    }
    try (FDBRecordContext context = openContext(hook)) {
        if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(storedRecords);
        } else {
            validateUsingNewerVersionFormat(storedRecords);
        }
    }
    // Update to the current format version
    formatVersion = FDBRecordStore.MAX_SUPPORTED_FORMAT_VERSION;
    if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION)) {
        // After format version upgrade, each record should now store that it has the version inlined
        storedRecords = storedRecords.stream().map(record -> new FDBStoredRecord<>(record.getPrimaryKey(), record.getRecordType(), record.getRecord(), record.getKeyCount() + 1, record.getKeySize() * 2 + 1, record.getValueSize() + 1 + FDBRecordVersion.VERSION_LENGTH, record.isSplit(), true, record.getVersion())).collect(Collectors.toList());
    }
    try (FDBRecordContext context = openContext(hook)) {
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(storedRecords);
        } else {
            validateUsingNewerVersionFormat(storedRecords);
        }
        for (FDBStoredRecord<Message> storedRecord : storedRecords) {
            Optional<FDBRecordVersion> loadedVersion = recordStore.loadRecordVersion(storedRecord.getPrimaryKey());
            assertTrue(loadedVersion.isPresent());
            assertEquals(storedRecord.getVersion(), loadedVersion.get());
            RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().equalsValue(storedRecord.getVersion())).build();
            RecordQueryPlan plan = planner.plan(query);
            final String endpointString = "[" + storedRecord.getVersion().toVersionstamp(false).toString() + "]";
            assertThat(plan, indexScan(allOf(indexName("globalVersion"), bounds(hasTupleString("[" + endpointString + "," + endpointString + "]")))));
            List<FDBStoredRecord<Message>> queriedRecords = recordStore.executeQuery(plan).map(FDBQueriedRecord::getStoredRecord).asList().join();
            assertEquals(Collections.singletonList(storedRecord), queriedRecords);
        }
        assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
        final List<FDBStoredRecord<Message>> fewerRecords = storedRecords.subList(1, storedRecords.size());
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(fewerRecords);
        } else {
            validateUsingNewerVersionFormat(fewerRecords);
        }
        recordStore.saveRecord(storedRecords.get(0).getRecord());
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(fewerRecords);
        } else {
            validateUsingNewerVersionFormat(fewerRecords);
        }
    // do not commit (so we can do a second upgrade)
    }
    final Index newValueIndex = new Index("MySimpleRecord$num2", field("num_value_2"), IndexTypes.VALUE);
    final Index newVersionIndex = new Index("MySimpleRecord$version-num2", concat(VersionKeyExpression.VERSION, field("num_value_2")), IndexTypes.VERSION);
    final RecordMetaDataHook hookWithNewIndexes = metaDataBuilder -> {
        hook.apply(metaDataBuilder);
        metaDataBuilder.addIndex("MySimpleRecord", newValueIndex);
        metaDataBuilder.addIndex("MySimpleRecord", newVersionIndex);
    };
    final List<FDBStoredRecord<Message>> newStoredRecords;
    try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
        assertTrue(recordStore.getRecordStoreState().isReadable(newValueIndex));
        boolean performedMigration = testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION && (splitLongRecords || testFormatVersion >= FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION);
        assertNotEquals(performedMigration, recordStore.getRecordStoreState().isReadable(newVersionIndex));
        if (recordStore.getRecordStoreState().isReadable(newVersionIndex)) {
            // Validate versions are the same for all records in index and in primary store
            List<Pair<Tuple, FDBRecordVersion>> recordScannedValues = recordStore.scanRecords(null, ScanProperties.FORWARD_SCAN).map(record -> Pair.of(record.getPrimaryKey(), record.getVersion())).asList().join();
            List<Pair<Tuple, FDBRecordVersion>> indexedScannedValues = recordStore.scanIndex(newVersionIndex, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(indexEntry -> Pair.of(TupleHelpers.subTuple(indexEntry.getKey(), 2, indexEntry.getKey().size()), FDBRecordVersion.fromVersionstamp(indexEntry.getKey().getVersionstamp(0), false))).asList().join();
            assertEquals(recordScannedValues, indexedScannedValues);
        }
        // Save record at newer version
        assertTrue(recordStore.deleteRecord(storedRecords.get(0).getPrimaryKey()));
        FDBStoredRecord<Message> newRecord0 = recordStore.saveRecord(storedRecords.get(0).getRecord());
        FDBStoredRecord<Message> newRecord2 = recordStore.saveRecord(storedRecords.get(2).getRecord());
        assertEquals(newRecord0.getVersion(), recordStore.loadRecordVersion(newRecord0.getPrimaryKey()).get());
        assertEquals(newRecord2.getVersion(), recordStore.loadRecordVersion(newRecord2.getPrimaryKey()).get());
        context.commit();
        byte[] versionstamp = context.getVersionStamp();
        newStoredRecords = Arrays.asList(newRecord0.withVersion(FDBRecordVersion.complete(versionstamp, 0)), storedRecords.get(1), newRecord2.withVersion(FDBRecordVersion.complete(versionstamp, 1)));
    }
    try (FDBRecordContext context = openContext(hookWithNewIndexes)) {
        if (!splitLongRecords && testFormatVersion < FDBRecordStore.SAVE_UNSPLIT_WITH_SUFFIX_FORMAT_VERSION) {
            validateUsingOlderVersionFormat(newStoredRecords);
        } else {
            validateUsingNewerVersionFormat(newStoredRecords);
        }
    }
}

Example 79 with RecordQueryPlan

use of com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan in project fdb-record-layer by FoundationDB.

the class TextIndexTest method performQueryWithIndexScan.

@Nonnull
private Set<Long> performQueryWithIndexScan(@Nonnull RecordMetaDataHook hook, @Nonnull Index index, @Nonnull QueryComponent filter) throws Exception {
    final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setTimeLimit(3000).build();
    final RecordQuery query = RecordQuery.newBuilder().setRecordType(SIMPLE_DOC).setRequiredResults(Collections.singletonList(field("doc_id"))).setFilter(filter).build();
    Set<Long> results = new HashSet<>();
    RecordQueryPlan plan;
    byte[] continuation;
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordQueryPlanner planner = new RecordQueryPlanner(recordStore.getRecordMetaData(), recordStore.getRecordStoreState());
        plan = planner.plan(query);
        assertThat(filter, instanceOf(FieldWithComparison.class));
        FieldWithComparison fieldFilter = (FieldWithComparison) filter;
        if (fieldFilter.getComparison() instanceof Comparisons.TextContainsAllPrefixesComparison && ((Comparisons.TextContainsAllPrefixesComparison) fieldFilter.getComparison()).isStrict()) {
            // Strict field field comparisons cannot be covering
            assertThat(plan, descendant(textIndexScan(indexName(index.getName()))));
        } else {
            assertThat(plan, descendant(coveringIndexScan(textIndexScan(indexName(index.getName())))));
        }
        try (RecordCursor<Long> cursor = recordStore.executeQuery(plan, null, executeProperties).map(record -> record.getPrimaryKey().getLong(0))) {
            cursor.forEach(results::add).get();
            continuation = cursor.getNext().getContinuation().toBytes();
        }
    }
    while (continuation != null) {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context, hook);
            try (RecordCursor<Long> cursor = recordStore.executeQuery(plan, continuation, executeProperties).map(record -> record.getPrimaryKey().getLong(0))) {
                cursor.forEach(results::add).get();
                continuation = cursor.getNext().getContinuation().toBytes();
            }
        }
    }
    return results;
}

Example 80 with RecordQueryPlan

use of com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan in project fdb-record-layer by FoundationDB.

the class TextIndexTest method queryDocumentsWithScanLimit.

@Test
public void queryDocumentsWithScanLimit() throws Exception {
    // Load a big (ish) data set
    final int recordCount = 100;
    final int batchSize = 10;
    for (int i = 0; i < recordCount; i += batchSize) {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context);
            for (int j = 0; j < batchSize; j++) {
                SimpleDocument document = SimpleDocument.newBuilder().setDocId(i + j).setText((i + j) % 2 == 0 ? "some" : "text").build();
                recordStore.saveRecord(document);
            }
            commit(context);
        }
    }
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        RecordQuery query = RecordQuery.newBuilder().setRecordType(SIMPLE_DOC).setFilter(Query.field("text").text().containsAll("some text")).build();
        RecordQueryPlan plan = planner.plan(query);
        boolean done = false;
        int totalKeysLoaded = 0;
        byte[] continuation = null;
        while (!done) {
            final int priorKeysLoaded = getLoadTextEntryCount(recordStore);
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(50).build();
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties);
            assertEquals(Collections.emptyList(), cursor.asList().get());
            RecordCursorResult<FDBQueriedRecord<Message>> noNextResult = cursor.getNext();
            assertThat(noNextResult.hasNext(), is(false));
            final int newKeysLoaded = getLoadTextEntryCount(recordStore);
            totalKeysLoaded += newKeysLoaded - priorKeysLoaded;
            if (!noNextResult.getNoNextReason().isSourceExhausted()) {
                assertEquals(50, newKeysLoaded - priorKeysLoaded);
                assertEquals(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED, noNextResult.getNoNextReason());
                assertNotNull(noNextResult.getContinuation().toBytes());
            } else {
                assertNull(noNextResult.getContinuation().toBytes());
                done = true;
            }
            continuation = noNextResult.getContinuation().toBytes();
        }
        assertEquals(recordCount + 2, totalKeysLoaded);
        commit(context);
    }
}