Examples with KeyExpression com.apple.foundationdb.record.metadata.expressions.KeyExpression used on opensource projects

Example 61 with KeyExpression

use of com.apple.foundationdb.record.metadata.expressions.KeyExpression in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreDeleteWhereTest method testDeleteWhereGroupedCount.

@Test
public void testDeleteWhereGroupedCount() throws Exception {
    KeyExpression groupExpr = concat(field("header").nest(field("rec_no")), field("header").nest(field("path")));
    RecordMetaDataHook hook = metaData -> {
        metaData.getRecordType("MyRecord").setPrimaryKey(concat(field("header").nest(field("rec_no")), field("header").nest(field("path")), field("header").nest(field("num"))));
        metaData.addUniversalIndex(new Index("MyRecord$groupedCount", new GroupingKeyExpression(groupExpr, 0), IndexTypes.COUNT));
        metaData.addUniversalIndex(new Index("MyRecord$groupedUpdateCount", new GroupingKeyExpression(groupExpr, 0), IndexTypes.COUNT_UPDATES));
    };
    try (FDBRecordContext context = openContext()) {
        openRecordWithHeader(context, hook);
        saveHeaderRecord(1, "a", 0, "lynx");
        saveHeaderRecord(1, "a", 1, "bobcat");
        saveHeaderRecord(1, "b", 2, "panther");
        saveHeaderRecord(2, "a", 3, "jaguar");
        saveHeaderRecord(2, "b", 4, "leopard");
        saveHeaderRecord(2, "c", 5, "lion");
        saveHeaderRecord(2, "d", 6, "tiger");
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        openRecordWithHeader(context, hook);
        // Number of records where first component of primary key is 1
        assertEquals(3, recordStore.getSnapshotRecordCount(groupExpr, Key.Evaluated.scalar(1)).join().longValue());
        // Number of updates to such records
        assertEquals(3, recordStore.getSnapshotRecordUpdateCount(groupExpr, Key.Evaluated.scalar(1)).join().longValue());
        recordStore.deleteRecordsWhere(Query.and(Query.field("header").matches(Query.field("rec_no").equalsValue(1)), Query.field("header").matches(Query.field("path").equalsValue("a"))));
        assertEquals(1, recordStore.getSnapshotRecordCount(groupExpr, Key.Evaluated.scalar(1)).join().longValue());
        // Deleting by group prefix resets the update counter for the group(s)
        assertEquals(1, recordStore.getSnapshotRecordUpdateCount(groupExpr, Key.Evaluated.scalar(1)).join().longValue());
        context.commit();
    }
}

Example 62 with KeyExpression

use of com.apple.foundationdb.record.metadata.expressions.KeyExpression in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method aggregateJoinIndex.

@Test
public void aggregateJoinIndex() throws Exception {
    final KeyExpression pkey = concat(recordType(), field("uuid"));
    metaDataBuilder.getRecordType("Customer").setPrimaryKey(pkey);
    metaDataBuilder.getRecordType("Order").setPrimaryKey(pkey);
    metaDataBuilder.getRecordType("Item").setPrimaryKey(pkey);
    metaDataBuilder.addIndex("Customer", "name");
    metaDataBuilder.addIndex("Order", "order_no");
    metaDataBuilder.addIndex("Order", "customer_uuid");
    metaDataBuilder.addIndex("Item", "order_uuid");
    final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("COI");
    joined.addConstituent("c", "Customer");
    joined.addConstituent("o", "Order");
    joined.addConstituent("i", "Item");
    joined.addJoin("o", "customer_uuid", "c", "uuid");
    joined.addJoin("i", "order_uuid", "o", "uuid");
    metaDataBuilder.addIndex(joined, new Index("total_price_by_city", field("i").nest("total_price").groupBy(field("c").nest("city")), IndexTypes.SUM));
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        TestRecordsJoinIndexProto.Customer.Builder c = TestRecordsJoinIndexProto.Customer.newBuilder();
        c.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setName("Jones").setCity("Boston");
        recordStore.saveRecord(c.build());
        c.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setName("Smith").setCity("New York");
        recordStore.saveRecord(c.build());
        c.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setName("Lee").setCity("Boston");
        recordStore.saveRecord(c.build());
        context.commit();
    }
    final RecordQuery findByName = RecordQuery.newBuilder().setRecordType("Customer").setFilter(Query.field("name").equalsParameter("name")).build();
    final RecordQuery findByOrderNo = RecordQuery.newBuilder().setRecordType("Order").setFilter(Query.field("order_no").equalsParameter("order_no")).build();
    final Index index = metaDataBuilder.getRecordMetaData().getIndex("total_price_by_city");
    final IndexAggregateFunction sumByCity = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    final List<String> coi = Collections.singletonList("COI");
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        TestRecordsJoinIndexProto.Customer.Builder c = TestRecordsJoinIndexProto.Customer.newBuilder();
        c.mergeFrom(recordStore.planQuery(findByName).execute(recordStore, EvaluationContext.forBinding("name", "Jones")).first().join().orElseThrow(() -> new RuntimeException("not found")).getRecord());
        TestRecordsJoinIndexProto.Order.Builder o = TestRecordsJoinIndexProto.Order.newBuilder();
        o.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setOrderNo(1001).setCustomerUuid(c.getUuid());
        recordStore.saveRecord(o.build());
        TestRecordsJoinIndexProto.Item.Builder i = TestRecordsJoinIndexProto.Item.newBuilder();
        i.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setItemNo(123).setQuantity(100).setTotalPrice(200).setOrderUuid(o.getUuid());
        recordStore.saveRecord(i.build());
        i.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setItemNo(456).setQuantity(10).setTotalPrice(1000).setOrderUuid(o.getUuid());
        recordStore.saveRecord(i.build());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        TestRecordsJoinIndexProto.Customer.Builder c = TestRecordsJoinIndexProto.Customer.newBuilder();
        c.mergeFrom(recordStore.planQuery(findByName).execute(recordStore, EvaluationContext.forBinding("name", "Smith")).first().join().orElseThrow(() -> new RuntimeException("not found")).getRecord());
        TestRecordsJoinIndexProto.Order.Builder o = TestRecordsJoinIndexProto.Order.newBuilder();
        o.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setOrderNo(1002).setCustomerUuid(c.getUuid());
        recordStore.saveRecord(o.build());
        TestRecordsJoinIndexProto.Item.Builder i = TestRecordsJoinIndexProto.Item.newBuilder();
        i.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setItemNo(789).setQuantity(20).setTotalPrice(200).setOrderUuid(o.getUuid());
        recordStore.saveRecord(i.build());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        TestRecordsJoinIndexProto.Customer.Builder c = TestRecordsJoinIndexProto.Customer.newBuilder();
        c.mergeFrom(recordStore.planQuery(findByName).execute(recordStore, EvaluationContext.forBinding("name", "Lee")).first().join().orElseThrow(() -> new RuntimeException("not found")).getRecord());
        TestRecordsJoinIndexProto.Order.Builder o = TestRecordsJoinIndexProto.Order.newBuilder();
        o.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setOrderNo(1003).setCustomerUuid(c.getUuid());
        recordStore.saveRecord(o.build());
        TestRecordsJoinIndexProto.Item.Builder i = TestRecordsJoinIndexProto.Item.newBuilder();
        i.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setItemNo(123).setQuantity(150).setTotalPrice(300).setOrderUuid(o.getUuid());
        recordStore.saveRecord(i.build());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        assertEquals(Tuple.from(1500), recordStore.evaluateAggregateFunction(coi, sumByCity, Key.Evaluated.scalar("Boston"), IsolationLevel.SERIALIZABLE).join());
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        TestRecordsJoinIndexProto.Customer.Builder c = TestRecordsJoinIndexProto.Customer.newBuilder();
        c.mergeFrom(recordStore.planQuery(findByName).execute(recordStore, EvaluationContext.forBinding("name", "Lee")).first().join().orElseThrow(() -> new RuntimeException("not found")).getRecord());
        TestRecordsJoinIndexProto.Order.Builder o = TestRecordsJoinIndexProto.Order.newBuilder();
        o.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setOrderNo(1004).setCustomerUuid(c.getUuid());
        recordStore.saveRecord(o.build());
        TestRecordsJoinIndexProto.Item.Builder i = TestRecordsJoinIndexProto.Item.newBuilder();
        i.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setItemNo(456).setQuantity(1).setTotalPrice(100).setOrderUuid(o.getUuid());
        recordStore.saveRecord(i.build());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        assertEquals(Tuple.from(1600), recordStore.evaluateAggregateFunction(coi, sumByCity, Key.Evaluated.scalar("Boston"), IsolationLevel.SERIALIZABLE).join());
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        TestRecordsJoinIndexProto.Order.Builder o = TestRecordsJoinIndexProto.Order.newBuilder();
        o.mergeFrom(recordStore.planQuery(findByOrderNo).execute(recordStore, EvaluationContext.forBinding("order_no", 1003)).first().join().orElseThrow(() -> new RuntimeException("not found")).getRecord());
        TestRecordsJoinIndexProto.Item.Builder i = TestRecordsJoinIndexProto.Item.newBuilder();
        i.setUuid(TupleFieldsHelper.toProto(UUID.randomUUID())).setItemNo(789).setQuantity(10).setTotalPrice(100).setOrderUuid(o.getUuid());
        recordStore.saveRecord(i.build());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        assertEquals(Tuple.from(1700), recordStore.evaluateAggregateFunction(coi, sumByCity, Key.Evaluated.scalar("Boston"), IsolationLevel.SERIALIZABLE).join());
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        TestRecordsJoinIndexProto.Customer.Builder c = TestRecordsJoinIndexProto.Customer.newBuilder();
        c.mergeFrom(recordStore.planQuery(findByName).execute(recordStore, EvaluationContext.forBinding("name", "Lee")).first().join().orElseThrow(() -> new RuntimeException("not found")).getRecord());
        c.setCity("San Francisco");
        recordStore.saveRecord(c.build());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        assertEquals(Tuple.from(1200), recordStore.evaluateAggregateFunction(coi, sumByCity, Key.Evaluated.scalar("Boston"), IsolationLevel.SERIALIZABLE).join());
        Map<Tuple, Tuple> expected = ImmutableMap.of(Tuple.from("Boston"), Tuple.from(1200), Tuple.from("New York"), Tuple.from(200), Tuple.from("San Francisco"), Tuple.from(500));
        Map<Tuple, Tuple> results = recordStore.scanIndex(index, IndexScanType.BY_GROUP, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().join().stream().collect(Collectors.toMap(IndexEntry::getKey, IndexEntry::getValue));
        assertEquals(expected, results);
    }
}

Example 63 with KeyExpression

use of com.apple.foundationdb.record.metadata.expressions.KeyExpression in project fdb-record-layer by FoundationDB.

the class LuceneIndexQueryPlan method getToPartialRecord.

/**
 * Get the {@link IndexKeyValueToPartialRecord} instance for an {@link IndexEntry} representing a result of Lucene auto-complete suggestion.
 * The partial record contains the suggestion in the field where it is indexed from, and the grouping keys if there are any.
 */
@VisibleForTesting
public static IndexKeyValueToPartialRecord getToPartialRecord(@Nonnull Index index, @Nonnull RecordType recordType, @Nonnull IndexScanType scanType) {
    final IndexKeyValueToPartialRecord.Builder builder = IndexKeyValueToPartialRecord.newBuilder(recordType);
    KeyExpression root = index.getRootExpression();
    if (root instanceof GroupingKeyExpression) {
        KeyExpression groupingKey = ((GroupingKeyExpression) root).getGroupingSubKey();
        for (int i = 0; i < groupingKey.getColumnSize(); i++) {
            AvailableFields.addCoveringField(groupingKey, AvailableFields.FieldData.of(IndexKeyValueToPartialRecord.TupleSource.KEY, i), builder);
        }
    }
    builder.addRequiredMessageFields();
    if (!builder.isValid(true)) {
        throw new RecordCoreException("Missing required field for result record").addLogInfo(LogMessageKeys.INDEX_NAME, index.getName()).addLogInfo(LogMessageKeys.RECORD_TYPE, recordType.getName()).addLogInfo(LogMessageKeys.SCAN_TYPE, scanType);
    }
    builder.addRegularCopier(new LuceneIndexKeyValueToPartialRecordUtils.LuceneAutoCompleteCopier());
    return builder.build();
}

Example 64 with KeyExpression

use of com.apple.foundationdb.record.metadata.expressions.KeyExpression in project fdb-record-layer by FoundationDB.

the class LucenePlanner method planLucene.

@Override
protected ScoredPlan planLucene(@Nonnull CandidateScan candidateScan, @Nonnull Index index, @Nonnull QueryComponent filter, @Nullable KeyExpression sort) {
    FilterSatisfiedMask filterMask = FilterSatisfiedMask.of(filter);
    KeyExpression rootExp = index.getRootExpression();
    ScanComparisons groupingComparisons;
    // Getting grouping information from the index key and query filter
    if (rootExp instanceof GroupingKeyExpression) {
        KeyExpression groupingKey = ((GroupingKeyExpression) rootExp).getGroupingSubKey();
        QueryToKeyMatcher.Match groupingMatch = new QueryToKeyMatcher(filter).matchesCoveringKey(groupingKey, filterMask);
        if (!groupingMatch.getType().equals((QueryToKeyMatcher.MatchType.EQUALITY))) {
            return null;
        }
        groupingComparisons = new ScanComparisons(groupingMatch.getEqualityComparisons(), Collections.emptySet());
    } else {
        groupingComparisons = null;
    }
    LuceneIndexQueryPlan lucenePlan = getComparisonsForLuceneFilter(index, null, filter, filterMask, groupingComparisons);
    if (lucenePlan == null) {
        return null;
    }
    RecordQueryPlan plan = lucenePlan;
    plan = addTypeFilterIfNeeded(candidateScan, plan, getPossibleTypes(index));
    if (filterMask.allSatisfied()) {
        filterMask.setSatisfied(true);
    }
    return new ScoredPlan(plan, filterMask.getUnsatisfiedFilters(), Collections.emptyList(), 11 - filterMask.getUnsatisfiedFilters().size(), lucenePlan.createsDuplicates(), null);
}

Example 65 with KeyExpression

use of com.apple.foundationdb.record.metadata.expressions.KeyExpression in project fdb-record-layer by FoundationDB.

the class LuceneIndexKeyValueToPartialRecordUtils method buildPartialRecord.

public static void buildPartialRecord(@Nonnull KeyExpression root, @Nonnull Descriptors.Descriptor descriptor, @Nonnull Message.Builder builder, @Nonnull String luceneField, @Nonnull String suggestion, @Nonnull Tuple groupingKey) {
    final KeyExpression expression = root instanceof GroupingKeyExpression ? ((GroupingKeyExpression) root).getWholeKey() : root;
    LuceneIndexExpressions.getFieldsRecursively(expression, new PartialRecordBuildSource(null, descriptor, builder), (source, fieldName, value, type, stored, overriddenKeyRanges, groupingKeyIndex) -> {
        if (groupingKeyIndex > -1) {
            if (groupingKeyIndex > groupingKey.size() - 1) {
                throw new RecordCoreException("Invalid grouping value tuple given a grouping key").addLogInfo(LogMessageKeys.VALUE, groupingKey.toString());
            }
            source.buildMessage(groupingKey.get(groupingKeyIndex), (String) value, null, null, false);
        } else if (type.equals(LuceneIndexExpressions.DocumentFieldType.TEXT)) {
            buildIfFieldNameMatch(source, fieldName, luceneField, overriddenKeyRanges, suggestion, (String) value);
        }
    }, null, 0, root instanceof GroupingKeyExpression ? ((GroupingKeyExpression) root).getGroupingCount() : 0, new ArrayList<>());
}