Examples with FDBQueriedRecord com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord used on opensource projects

Example 81 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method nestedAndQuery.

@Test
void nestedAndQuery() {
    final KeyExpression num_by_str = field("nested").nest(field("entry", FanOut).nest(concatenateFields("str_value", "num_value")));
    final GroupingKeyExpression nested_num_by_str = concat(field("num_value_1"), num_by_str).group(1);
    final KeyExpression nested_num_by_str_num2 = concat(field("num_value_1"), field("num_value_2"), num_by_str).group(1);
    final KeyExpression nested_num_by_str_num3 = concat(field("num_value_1"), field("num_value_3"), num_by_str).group(1);
    final RecordMetaDataHook nested_rec_no_by_str_nums_hook = metadata -> {
        final RecordTypeBuilder recordType = metadata.getRecordType("MyNestedRecord");
        metadata.addIndex(recordType, new Index("nested_num_by_str_num2", nested_num_by_str_num2, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
        metadata.addIndex(recordType, new Index("nested_num_by_str_num3", nested_num_by_str_num3, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
    };
    final IndexAggregateFunctionCall bitmap_value_nested_num_by_str = new IndexAggregateFunctionCall(FunctionNames.BITMAP_VALUE, nested_num_by_str);
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
        for (int recNo = 100; recNo < 200; recNo++) {
            recordStore.saveRecord(TestRecordsBitmapProto.MyNestedRecord.newBuilder().setRecNo(recNo).setNumValue1(1).setNested(TestRecordsBitmapProto.MyNestedRecord.Nested.newBuilder().addEntry(TestRecordsBitmapProto.MyNestedRecord.Nested.Entry.newBuilder().setStrValue((recNo & 1) == 1 ? "odd" : "even").setNumValue(recNo + 1000))).setNumValue2(recNo % 7).setNumValue3(recNo % 5).build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
        setupPlanner(null);
        final RecordQuery recordQuery = RecordQuery.newBuilder().setRecordType("MyNestedRecord").setFilter(Query.and(Query.field("num_value_1").equalsValue(1), Query.field("nested").matches(Query.field("entry").oneOfThem().matches(Query.field("str_value").equalsValue("odd"))), Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4))).setRequiredResults(Collections.singletonList(field("nested").nest(field("entry", FanOut).nest("num_value")))).build();
        final RecordQueryPlan queryPlan = ComposedBitmapIndexAggregate.tryPlan((RecordQueryPlanner) planner, recordQuery, bitmap_value_nested_num_by_str, IndexQueryabilityFilter.DEFAULT).orElseGet(() -> fail("Cannot plan query"));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 3, odd],[1, 3, odd]]"))))), coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 4, odd],[1, 4, odd]]"))))))));
        assertEquals(1000204717, queryPlan.planHash());
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && (i % 5) == 4).map(i -> i + 1000).collect(Collectors.toList())));
    }
}

Example 82 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method andQuery.

@Test
void andQuery() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        saveRecords(100, 200);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        // Covering(Index(rec_no_by_str_num2 [[odd, 3],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 [[odd, 4],[odd, 4]] BY_GROUP) -> [rec_no: KEY[2]])
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4)));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 3],[odd, 3]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 4],[odd, 4]]"))))))));
        assertEquals(1339577615, queryPlan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1022755654, queryPlan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(944107193, queryPlan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && (i % 5) == 4).collect(Collectors.toList())));
    }
}

Example 83 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method nonOverlappingOrQuery.

@Test
void nonOverlappingOrQuery() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        for (int recNo = 100; recNo < 200; recNo++) {
            recordStore.saveRecord(TestRecordsBitmapProto.MySimpleRecord.newBuilder().setRecNo(recNo).setStrValue((recNo & 1) == 1 ? "odd" : "even").setNumValue2(1).build());
        }
        for (int recNo = 500; recNo < 600; recNo++) {
            recordStore.saveRecord(TestRecordsBitmapProto.MySimpleRecord.newBuilder().setRecNo(recNo).setStrValue((recNo & 1) == 1 ? "odd" : "even").setNumValue3(1).build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        // Covering(Index(rec_no_by_str_num2 [[odd, 1],[odd, 1]] BY_GROUP) -> [rec_no: KEY[2]]) BITOR Covering(Index(rec_no_by_str_num3 [[odd, 1],[odd, 1]] BY_GROUP) -> [rec_no: KEY[2]])
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.or(Query.field("num_value_2").equalsValue(1), Query.field("num_value_3").equalsValue(1))));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITOR [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 1],[odd, 1]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 1],[odd, 1]]"))))))));
        assertEquals(-556720460, queryPlan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1315884767, queryPlan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-952190817, queryPlan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.concat(IntStream.range(100, 200), IntStream.range(500, 600)).boxed().filter(i -> (i & 1) == 1).collect(Collectors.toList())));
    }
}

Example 84 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method testPlansReturnSameRecordsRegardlessOfLimit.

@ParameterizedTest(name = "plansByContinuation() [{index}] {0}")
@MethodSource("plans")
public void testPlansReturnSameRecordsRegardlessOfLimit(String description, boolean notUsed, RecordQueryPlan plan) throws Exception {
    setupSimpleRecordStore();
    final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
        TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
        record.mergeFrom(r.getRecord());
        return record.getRecNo();
    };
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final List<Long> allAtOnce;
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            allAtOnce = cursor.map(getRecNo).asList().get();
        }
        for (long byteLimit = 0; byteLimit < 1000; byteLimit += 100) {
            final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteLimit).build();
            final List<Long> byContinuation = new ArrayList<>(allAtOnce.size());
            byte[] continuation = null;
            do {
                try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
                    RecordCursorResult<Long> result;
                    do {
                        result = cursor.onNext().get().map(getRecNo);
                        if (result.hasNext()) {
                            byContinuation.add(result.get());
                        }
                    } while (result.hasNext());
                    continuation = result.getContinuation().toBytes();
                }
            } while (continuation != null);
            assertEquals(allAtOnce, byContinuation);
        }
    }
}

Example 85 with FDBQueriedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method assertPlanLimitsWithCorrectExecution.

/**
 * Make detailed assertions about the number of bytes scanned when the byte scan limit is very close to the number
 * of records scanned between records.
 *
 * This helper method attempts to verify that the number of bytes being scanned during query execution is not too
 * much higher than the limit specified by {@link ExecuteProperties.Builder#setScannedBytesLimit(long)}. To do this,
 * it's given a list of the number of bytes scanned between individual records produced by the given plan. Then,
 * it sets the byte limit to that number of bytes, possibly increased or decreased by 1.
 *
 * Suppose that A, B, and C are successive records produced by the plan's cursor. Consider the execution of that
 * cursor in the absence of any limits. Let the number of bytes scanned between A and B be {@code m} and the number
 * of bytes scanned between B and C be {@code n}. This test relies on the following claims holding:
 * a) {@code m} and {@code n} are constant with respect to the asynchronous execution order of the underlying cursors.
 * b) If the byte scan limit is less than or equal to {@code m}, then at most {@code 2 * m} bytes are scanned between
 *   A and B, even if we resume the cursor by continuation. The same holds for {@code n}, B and C.
 * c) If the byte scan limit is {@code m + 1}, then at least {@code m} and at most {@code m + n} bytes are scanned.
 *
 * Note that claim (b) requires the factor of two because resuming certain cursors (such as a
 * {@link com.apple.foundationdb.record.provider.foundationdb.cursors.UnionCursor}) from a continuation requires
 * rescanning records that have already been deserialized.
 *
 * These claims should hold for most query plans. Notably, even claim (a) does not hold for the
 * {@link com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan}.
 * @param byteCountsByRecord a list of the number of bytes scanned between successive records without any limits
 * @param context an open record store context
 * @param plan a query plan to execute
 */
private void assertPlanLimitsWithCorrectExecution(@Nonnull List<Long> byteCountsByRecord, @Nonnull FDBRecordContext context, @Nonnull RecordQueryPlan plan) {
    byte[] continuation = null;
    int i = 0;
    context.getTimer().reset();
    while (i < byteCountsByRecord.size()) {
        final int currentIndex = i;
        // If the limit is slightly too low to scan the next record, we should scan it anyway and then stop.
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteCountsByRecord.get(currentIndex) - 1).build();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            final long bytesScanned = byteCounter.getBytesScanned(context);
            if (currentIndex == byteCountsByRecord.size() - 1) {
                // Final record
                // If this is a final record, then there must be enough room to scan everything until the end,
                // because of how the byteCountsByRecord are counted.
                assertTrue(result.hasNext());
            }
            // cursors) then we might not have a result when we expect to.
            if (result.hasNext()) {
                i++;
                context.getTimer().reset();
            } else {
                // Check that we stopped because of the byte scan limit.
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
            // Assertion of claim (b)
            assertThat(bytesScanned, lessThanOrEqualTo(2 * byteCountsByRecord.get(currentIndex)));
            continuation = result.getContinuation().toBytes();
        }
    }
    continuation = null;
    i = 0;
    while (i < byteCountsByRecord.size()) {
        final int currentIndex = i;
        // If the limit is exactly right we should scan the record and then stop.
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteCountsByRecord.get(currentIndex)).build();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            final long bytesScanned = byteCounter.getBytesScanned(context);
            // cursors) then we might not have a result when we expect to.
            if (result.hasNext()) {
                i++;
                context.getTimer().reset();
            } else if (currentIndex < byteCountsByRecord.size() - 1) {
                // not the final record yet
                // Check that we stopped because of the byte scan limit.
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
            // Assertion of claim (b)
            assertThat(bytesScanned, lessThanOrEqualTo(2 * byteCountsByRecord.get(currentIndex)));
            continuation = result.getContinuation().toBytes();
        }
    }
    continuation = null;
    i = 0;
    while (i < byteCountsByRecord.size()) {
        // If the limit is slightly too high we should scan the record, proceed to the next one (if it exists), and then stop.
        final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteCountsByRecord.get(i) + 1).build();
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            assertTrue(result.hasNext());
            continuation = result.getContinuation().toBytes();
            result = cursor.getNext();
            long bytesScanned = byteCounter.getBytesScanned(context);
            // Assertion of claim (c)
            assertThat(bytesScanned, greaterThanOrEqualTo(byteCountsByRecord.get(i)));
            if (i < byteCountsByRecord.size() - 1) {
                // Assertion of claim (c)
                assertThat(bytesScanned, lessThanOrEqualTo(byteCountsByRecord.get(i) + byteCountsByRecord.get(i + 1)));
            }
            i++;
            if (result.hasNext()) {
                result = cursor.getNext();
                assertFalse(result.hasNext());
            }
            context.getTimer().reset();
        }
    }
}