Examples with QueryComponent com.apple.foundationdb.record.query.expressions.QueryComponent used on opensource projects

Example 16 with QueryComponent

use of com.apple.foundationdb.record.query.expressions.QueryComponent in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreQueryTest method queryWithContinuation.

/**
 * Verify that simple queries execute properly with continuations.
 */
@DualPlannerTest
void queryWithContinuation() throws Exception {
    setupSimpleRecordStore(null, (i, builder) -> {
        builder.setRecNo(i);
        builder.setNumValue2(i % 2);
        builder.setStrValueIndexed((i % 2 == 0) ? "even" : "odd");
    });
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, null);
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setAllowedIndexes(Collections.emptyList()).build();
        // Scan(<,>) | [MySimpleRecord]
        RecordQueryPlan plan = planner.plan(query);
        assertMatchesExactly(plan, typeFilterPlan(scanPlan().where(scanComparisons(unbounded()))).where(recordTypes(containsAll(ImmutableSet.of("MySimpleRecord")))));
        assertEquals(1623132336, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1955010341, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1955010341, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        byte[] continuation = null;
        List<TestRecords1Proto.MySimpleRecord> retrieved = new ArrayList<>(100);
        while (true) {
            RecordCursor<TestRecords1Proto.MySimpleRecord> cursor = recordStore.executeQuery(plan, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build());
            List<TestRecords1Proto.MySimpleRecord> list = cursor.asList().get();
            assertEquals(Math.min(10, 100 - retrieved.size()), list.size());
            for (int i = 0; i < list.size(); i++) {
                assertEquals(retrieved.size() + i, list.get(i).getRecNo());
            }
            assertDiscardedNone(context);
            retrieved.addAll(list);
            if (retrieved.size() > 100) {
                fail("added more records than were present");
            }
            continuation = cursor.getNext().getContinuation().toBytes();
            if (continuation == null) {
                break;
            }
        }
        query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("odd")).build();
        // Index(MySimpleRecord$str_value_indexed [[odd],[odd]])
        plan = planner.plan(query);
        assertMatchesExactly(plan, indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(range("[[odd],[odd]]"))));
        assertEquals(-1917280682, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1357054180, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(9136435, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        continuation = null;
        retrieved = new ArrayList<>(50);
        while (true) {
            RecordCursor<TestRecords1Proto.MySimpleRecord> cursor = recordStore.executeQuery(plan, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(5).build()).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build());
            List<TestRecords1Proto.MySimpleRecord> list = cursor.asList().get();
            assertEquals(Math.min(5, 50 - retrieved.size()), list.size());
            for (int i = 0; i < list.size(); i++) {
                assertEquals(2 * (retrieved.size() + i) + 1, list.get(i).getRecNo());
            }
            assertDiscardedNone(context);
            retrieved.addAll(list);
            if (retrieved.size() > 50) {
                fail("added more records than met filter");
            }
            continuation = cursor.getNext().getContinuation().toBytes();
            if (continuation == null) {
                break;
            }
        }
        clearStoreCounter(context);
        final QueryComponent filter = Query.field("num_value_2").equalsValue(0);
        query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).build();
        plan = planner.plan(query);
        if (planner instanceof RecordQueryPlanner) {
            assertMatchesExactly(plan, filterPlan(typeFilterPlan(scanPlan().where(scanComparisons(unbounded())))).where(queryComponents(exactly(equalsObject(filter)))));
            assertEquals(913370522, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        // TODO: https://github.com/FoundationDB/fdb-record-layer/issues/1074
        // assertEquals(389700036, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        } else {
            assertMatchesExactly(plan, predicatesFilterPlan(typeFilterPlan(scanPlan().where(scanComparisons(unbounded())))).where(predicates(only(valuePredicate(fieldValue(anyValue(), "num_value_2"), new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 0))))));
            assertEquals(-1244637277, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        }
        continuation = null;
        retrieved = new ArrayList<>(50);
        while (true) {
            RecordCursor<TestRecords1Proto.MySimpleRecord> cursor = recordStore.executeQuery(plan, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(15).build()).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build());
            List<TestRecords1Proto.MySimpleRecord> list = cursor.asList().get();
            assertEquals(Math.min(15, 50 - retrieved.size()), list.size());
            for (int i = 0; i < list.size(); i++) {
                assertEquals(2 * (retrieved.size() + i), list.get(i).getRecNo());
            }
            retrieved.addAll(list);
            if (retrieved.size() > 50) {
                fail("added more records than met filter");
            }
            continuation = cursor.getNext().getContinuation().toBytes();
            if (continuation == null) {
                break;
            }
        }
        assertDiscardedAtMost(51, context);
    }
}

Example 17 with QueryComponent

use of com.apple.foundationdb.record.query.expressions.QueryComponent in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method testInQueryIndexSortedDifferently.

/**
 * Verify that an IN against an unsorted list with an index is not implemented as an IN JOIN when the query sort is
 * not by the field with an IN filter.
 */
@DualPlannerTest
void testInQueryIndexSortedDifferently() throws Exception {
    complexQuerySetup(NO_HOOK);
    final QueryComponent filter = Query.field("num_value_3_indexed").in(asList(1, 4, 2));
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).setSort(field("str_value_indexed")).build();
    // Index(MySimpleRecord$str_value_indexed <,>) | num_value_3_indexed IN [1, 4, 2]
    RecordQueryPlan plan = planner.plan(query);
    // IN join is cancelled on account of incompatible sorting.
    if (planner instanceof RecordQueryPlanner) {
        assertMatchesExactly(plan, filterPlan(selfOrDescendantPlans(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded())))).where(queryComponents(exactly(equalsObject(filter)))));
        assertEquals(1775865786, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(972267, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(212572525, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, predicatesFilterPlan(selfOrDescendantPlans(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded())))).where(predicates(valuePredicate(fieldValue("num_value_3_indexed"), new Comparisons.ListComparison(Comparisons.Type.IN, ImmutableList.of(1, 4, 2))))));
        assertEquals(1470982333, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(800585401, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1012185659, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(60, querySimpleRecordStore(NO_HOOK, plan, EvaluationContext::empty, record -> assertThat(record.getNumValue3Indexed(), anyOf(is(1), is(2), is(4))), context -> TestHelpers.assertDiscardedAtMost(40, context)));
}

Example 18 with QueryComponent

use of com.apple.foundationdb.record.query.expressions.QueryComponent in project fdb-record-layer by FoundationDB.

the class FDBNestedFieldQueryTest method nestedRankMap.

/**
 * Verify that a rank index on a map-like repeated nested message can be scanned for rank comparisons.
 */
@Test
public void nestedRankMap() throws Exception {
    final GroupingKeyExpression rankGroup = new GroupingKeyExpression(concat(field("other_id"), field("map").nest(field("entry", KeyExpression.FanType.FanOut).nest(concatenateFields("key", "value")))), 1);
    final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsNestedMapProto.getDescriptor());
    metaDataBuilder.addIndex("OuterRecord", new Index("rank_value_by_key", rankGroup, IndexTypes.RANK));
    // TODO: This is not a very obvious way to specify this. But we don't have correlation names.
    final QueryComponent keyCondition = Query.field("map").matches(Query.field("entry").oneOfThem().matches(Query.field("key").equalsValue("alpha")));
    final QueryRecordFunction<Long> rank = Query.rank(rankGroup).withAdditionalCondition(keyCondition);
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaDataBuilder.getRecordMetaData());
        TestRecordsNestedMapProto.OuterRecord.Builder builder = TestRecordsNestedMapProto.OuterRecord.newBuilder().setOtherId(1);
        TestRecordsNestedMapProto.MapRecord.Builder mapBuilder = builder.getMapBuilder();
        builder.setRecId(1);
        mapBuilder.addEntryBuilder().setKey("alpha").setValue("abc");
        mapBuilder.addEntryBuilder().setKey("beta").setValue("bcd");
        recordStore.saveRecord(builder.build());
        builder.setRecId(2);
        mapBuilder.clear();
        mapBuilder.addEntryBuilder().setKey("alpha").setValue("aaa");
        mapBuilder.addEntryBuilder().setKey("beta").setValue("bbb");
        recordStore.saveRecord(builder.build());
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("OuterRecord").setFilter(Query.and(Query.field("other_id").equalsValue(1L), Query.rank(rankGroup).lessThan(10L), keyCondition)).build();
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, primaryKeyDistinct(indexScan(allOf(indexName("rank_value_by_key"), indexScanType(IndexScanType.BY_RANK), bounds(hasTupleString("([1, alpha, null],[1, alpha, 10])"))))));
    assertEquals(1307013946, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(-1725407749, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(825274646, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaDataBuilder.getRecordMetaData());
        try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
            RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
            assertTrue(result.hasNext());
            assertEquals(Tuple.from(2), result.get().getPrimaryKey());
            result = cursor.getNext();
            assertTrue(result.hasNext());
            assertEquals(Tuple.from(1), result.get().getPrimaryKey());
            assertEquals(1, rank.eval(recordStore, EvaluationContext.EMPTY, result.get().getStoredRecord()).get());
            result = cursor.getNext();
            assertFalse(result.hasNext());
        }
    }
}

Example 19 with QueryComponent

use of com.apple.foundationdb.record.query.expressions.QueryComponent in project fdb-record-layer by FoundationDB.

the class FDBInQueryTest method inQueryWithSortBySecondFieldOfCompoundIndex.

/**
 * Verify that an IN query with a sort can be implemented as an ordered union of compound indexes that can satisfy
 * the sort once the equality predicates from the IN have been pushed onto the indexes.
 * @see com.apple.foundationdb.record.query.plan.planning.InExtractor#asOr()
 */
@ParameterizedTest
@EnumSource(InAsOrUnionMode.class)
void inQueryWithSortBySecondFieldOfCompoundIndex(InAsOrUnionMode inAsOrMode) throws Exception {
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", "compoundIndex", concat(field("num_value_3_indexed"), field("str_value_indexed")));
    complexQuerySetup(hook);
    final List<Integer> inList = asList(1, 4, 2);
    final QueryComponent filter = Query.field("num_value_3_indexed").in(inList);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(filter).setSort(field("str_value_indexed")).build();
    // The configuration is planner-specific.
    assertTrue(planner instanceof RecordQueryPlanner);
    planner.setConfiguration(inAsOrMode.configure(planner.getConfiguration().asBuilder()).build());
    // Index(MySimpleRecord$str_value_indexed <,>) | num_value_3_indexed IN [1, 4, 2]
    // Index(compoundIndex [[1],[1]]) ∪[Field { 'str_value_indexed' None}, Field { 'rec_no' None}] Index(compoundIndex [[4],[4]]) ∪[Field { 'str_value_indexed' None}, Field { 'rec_no' None}] Index(compoundIndex [[2],[2]])
    // ∪(__in_num_value_3_indexed__0 IN [1, 4, 2]) Index(compoundIndex [EQUALS $__in_num_value_3_indexed__0])
    RecordQueryPlan plan = planner.plan(query);
    if (inAsOrMode == InAsOrUnionMode.AS_OR) {
        assertMatchesExactly(plan, unionPlan(inList.stream().map(number -> indexPlan().where(indexName("compoundIndex")).and(scanComparisons(range(String.format("[[%d],[%d]]", number, number))))).collect(ImmutableList.toImmutableList())).where(comparisonKey(concat(field("str_value_indexed"), primaryKey("MySimpleRecord")))));
        assertEquals(-1813975352, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1188407258, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(2089555085, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else if (inAsOrMode == InAsOrUnionMode.AS_UNION) {
        assertMatchesExactly(plan, inUnionPlan(indexPlan().where(indexName("compoundIndex")).and(scanComparisons(range("[EQUALS $__in_num_value_3_indexed__0]")))).where(inUnionComparisonKey(concat(field("str_value_indexed"), primaryKey("MySimpleRecord")))).and(inUnionValuesSources(exactly(inUnionInValues(equalsObject(inList)).and(inUnionBindingName("__in_num_value_3_indexed__0"))))));
        assertEquals(406364040, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(494706693, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-628125702, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertMatchesExactly(plan, filterPlan(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")).and(scanComparisons(unbounded()))).where(queryComponents(exactly(equalsObject(filter)))));
        assertEquals(1775865786, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(972267, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(212572525, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    assertEquals(60, querySimpleRecordStore(hook, plan, EvaluationContext::empty, record -> assertThat(record.getNumValue3Indexed(), anyOf(is(1), is(2), is(4))), context -> TestHelpers.assertDiscardedAtMost(40, context)));
}

Example 20 with QueryComponent

use of com.apple.foundationdb.record.query.expressions.QueryComponent in project fdb-record-layer by FoundationDB.

the class TextIndexTest method performQueryWithRecordStoreScan.

@Nonnull
private Set<Long> performQueryWithRecordStoreScan(@Nonnull RecordMetaDataHook hook, @Nonnull QueryComponent filter) throws Exception {
    final ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setTimeLimit(3000).build());
    Set<Long> results = new HashSet<>();
    byte[] continuation = null;
    do {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context);
            try (RecordCursor<Long> cursor = recordStore.scanRecords(continuation, scanProperties).filter(record -> record.getRecordType().getName().equals(SIMPLE_DOC)).filter(record -> filter.eval(recordStore, EvaluationContext.EMPTY, record) == Boolean.TRUE).map(record -> record.getPrimaryKey().getLong(0))) {
                cursor.forEach(results::add).get();
                RecordCursorResult<Long> noNextResult = cursor.getNext();
                continuation = noNextResult.getContinuation().toBytes();
            }
        }
    } while (continuation != null);
    return results;
}