Examples with ThenKeyExpression com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression used on opensource projects

Example 11 with ThenKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxValueNegative.

@Test
public void minMaxValueNegative() throws Exception {
    final FieldKeyExpression strValue = field("str_value_indexed");
    final FieldKeyExpression numValue2 = field("num_value_2");
    final FieldKeyExpression numValue3 = field("num_value_3_indexed");
    final ThenKeyExpression compound = concat(strValue, numValue2, numValue3);
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("compound", compound, IndexTypes.VALUE));
    };
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        final IndexAggregateFunction minValue2 = new IndexAggregateFunction(FunctionNames.MIN, numValue2, null);
        assertThrows(RecordCoreException.class, () -> {
            recordStore.evaluateAggregateFunction(types, minValue2, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join();
        });
        final IndexAggregateFunction minValue3GroupedIncorrectly = new IndexAggregateFunction(FunctionNames.MIN, numValue3.groupBy(numValue2, strValue), null);
        assertThrows(RecordCoreException.class, () -> {
            recordStore.evaluateAggregateFunction(types, minValue3GroupedIncorrectly, Key.Evaluated.concatenate(1, "foo"), IsolationLevel.SNAPSHOT).join();
        });
        final IndexAggregateFunction minValue3GroupedTooMany = new IndexAggregateFunction(FunctionNames.MIN, concat(numValue3, numValue2).groupBy(strValue), null);
        assertThrows(RecordCoreException.class, () -> {
            recordStore.evaluateAggregateFunction(types, minValue3GroupedTooMany, Key.Evaluated.scalar("foo"), IsolationLevel.SNAPSHOT).join();
        });
        commit(context);
    }
}

Example 12 with ThenKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxTupleGrouped.

@Test
public void minMaxTupleGrouped() throws Exception {
    final ThenKeyExpression tupleKey = concat(field("str_value_indexed"), field("num_value_2"));
    final GroupingKeyExpression byKey = tupleKey.groupBy(field("num_value_3_indexed"));
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("min", byKey, IndexTypes.MIN_EVER_TUPLE));
        md.addIndex(type, new Index("max", byKey, IndexTypes.MAX_EVER_TUPLE));
    };
    final IndexAggregateFunction minOverall = new IndexAggregateFunction(FunctionNames.MIN_EVER, tupleKey, null);
    final IndexAggregateFunction maxOverall = new IndexAggregateFunction(FunctionNames.MAX_EVER, tupleKey, null);
    final IndexAggregateFunction minByKey = new IndexAggregateFunction(FunctionNames.MIN_EVER, byKey, null);
    final IndexAggregateFunction maxByKey = new IndexAggregateFunction(FunctionNames.MAX_EVER, byKey, null);
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertNull(recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setNumValue3Indexed(i % 3);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            recBuilder.setNumValue2(i / 2);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(Tuple.from("even", 0), recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from("odd", 49), recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from("even", 2), recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from("odd", 48), recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        commit(context);
    }
}

Example 13 with ThenKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression in project fdb-record-layer by FoundationDB.

the class LuceneIndexExpressions method getFieldsRecursively.

@SuppressWarnings("squid:S3776")
public static <T extends RecordSource<T>> void getFieldsRecursively(@Nonnull KeyExpression expression, @Nonnull T source, @Nonnull DocumentDestination<T> destination, @Nullable String fieldNamePrefix, int keyIndex, int groupingCount, @Nonnull List<Integer> overriddenKeyRanges) {
    if (expression instanceof ThenKeyExpression) {
        int count = 0;
        for (KeyExpression child : ((ThenKeyExpression) expression).getChildren()) {
            getFieldsRecursively(child, source, destination, fieldNamePrefix, keyIndex + count, groupingCount, overriddenKeyRanges);
            count += child.getColumnSize();
        }
        return;
    }
    String fieldNameSuffix = null;
    boolean suffixOverride = false;
    if (expression instanceof LuceneFunctionKeyExpression.LuceneFieldName) {
        LuceneFunctionKeyExpression.LuceneFieldName fieldNameExpression = (LuceneFunctionKeyExpression.LuceneFieldName) expression;
        KeyExpression nameExpression = fieldNameExpression.getNameExpression();
        if (nameExpression instanceof LiteralKeyExpression) {
            fieldNameSuffix = (String) ((LiteralKeyExpression<?>) nameExpression).getValue();
        } else if (nameExpression instanceof FieldKeyExpression) {
            Iterator<Object> names = source.getValues((FieldKeyExpression) nameExpression).iterator();
            if (names.hasNext()) {
                fieldNameSuffix = (String) names.next();
                if (names.hasNext()) {
                    throw new RecordCoreException("Lucene field name override should evaluate to single value");
                }
            }
        } else {
            throw new RecordCoreException("Lucene field name override should be a literal or a field");
        }
        suffixOverride = true;
        expression = fieldNameExpression.getNamedExpression();
    }
    if (expression instanceof NestingKeyExpression) {
        NestingKeyExpression nestingExpression = (NestingKeyExpression) expression;
        FieldKeyExpression parentExpression = nestingExpression.getParent();
        KeyExpression child = nestingExpression.getChild();
        if (!suffixOverride) {
            fieldNameSuffix = parentExpression.getFieldName();
        } else {
            addOverriddenKeyRange(overriddenKeyRanges, fieldNamePrefix, fieldNameSuffix);
        }
        String fieldName = appendFieldName(fieldNamePrefix, fieldNameSuffix);
        for (T subsource : source.getChildren(parentExpression)) {
            getFieldsRecursively(child, subsource, destination, fieldName, keyIndex, groupingCount, overriddenKeyRanges);
        }
        if (suffixOverride) {
            // Remove the last 2 numbers added above
            removedLastOverriddenKeyRange(overriddenKeyRanges);
        }
        return;
    }
    boolean fieldStored = false;
    boolean fieldText = false;
    while (true) {
        if (expression instanceof LuceneFunctionKeyExpression.LuceneStored) {
            LuceneFunctionKeyExpression.LuceneStored storedExpression = (LuceneFunctionKeyExpression.LuceneStored) expression;
            fieldStored = true;
            expression = storedExpression.getStoredExpression();
        } else if (expression instanceof LuceneFunctionKeyExpression.LuceneText) {
            LuceneFunctionKeyExpression.LuceneText textExpression = (LuceneFunctionKeyExpression.LuceneText) expression;
            fieldText = true;
            expression = textExpression.getFieldExpression();
        } else {
            // TODO: More text options.
            break;
        }
    }
    if (expression instanceof FieldKeyExpression) {
        FieldKeyExpression fieldExpression = (FieldKeyExpression) expression;
        if (!suffixOverride) {
            fieldNameSuffix = fieldExpression.getFieldName();
        } else {
            addOverriddenKeyRange(overriddenKeyRanges, fieldNamePrefix, fieldNameSuffix);
        }
        String fieldName = appendFieldName(fieldNamePrefix, fieldNameSuffix);
        if (fieldName == null) {
            fieldName = "_";
        }
        Descriptors.Descriptor recordDescriptor = source.getDescriptor();
        Descriptors.FieldDescriptor fieldDescriptor = recordDescriptor.findFieldByName(fieldExpression.getFieldName());
        DocumentFieldType fieldType;
        if (fieldText) {
            switch(fieldDescriptor.getJavaType()) {
                case STRING:
                    fieldType = DocumentFieldType.TEXT;
                    break;
                default:
                    throw new RecordCoreException("Unknown Lucene text field type");
            }
        } else {
            switch(fieldDescriptor.getJavaType()) {
                case STRING:
                    fieldType = DocumentFieldType.STRING;
                    break;
                case INT:
                    fieldType = DocumentFieldType.INT;
                    break;
                case LONG:
                    fieldType = DocumentFieldType.LONG;
                    break;
                case DOUBLE:
                    fieldType = DocumentFieldType.DOUBLE;
                    break;
                case BOOLEAN:
                    fieldType = DocumentFieldType.BOOLEAN;
                    break;
                default:
                    throw new RecordCoreException("Unknown Lucene field type");
            }
        }
        for (Object value : source.getValues(fieldExpression)) {
            destination.addField(source, fieldName, value, fieldType, fieldStored, overriddenKeyRanges, keyIndex < groupingCount ? keyIndex : -1);
        }
        if (suffixOverride) {
            // Remove the last 2 numbers added above
            removedLastOverriddenKeyRange(overriddenKeyRanges);
        }
        return;
    }
    throw new RecordCoreException("Unknown Lucene field key expression");
}

Example 14 with ThenKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression in project fdb-record-layer by FoundationDB.

the class LuceneDocumentFromRecord method getGroupedFields.

// Grouping keys are evaluated more or less normally, turning into multiple groups.
// Each group corresponds to a single document in a separate index / directory.
// Within that document, the grouped fields are merged.
protected static <M extends Message> void getGroupedFields(@Nonnull List<KeyExpression> keys, int keyIndex, int keyPosition, int groupingCount, @Nonnull Tuple groupPrefix, @Nonnull FDBRecord<M> rec, @Nonnull Message message, @Nonnull Map<Tuple, List<DocumentField>> result, @Nullable String fieldNamePrefix) {
    if (keyIndex >= keys.size()) {
        return;
    }
    KeyExpression key = keys.get(keyIndex);
    int keySize = key.getColumnSize();
    if (keyPosition + keySize <= groupingCount) {
        // Entirely in the grouping portion: extend group prefix with normal evaluation.
        List<Key.Evaluated> groups = key.evaluateMessage(rec, message);
        for (Key.Evaluated group : groups) {
            Tuple wholeGroup = groupPrefix.addAll(group.toTupleAppropriateList());
            if (groupingCount == wholeGroup.size()) {
                result.putIfAbsent(wholeGroup, new ArrayList<>());
            }
            getGroupedFields(keys, keyIndex + 1, keyPosition + key.getColumnSize(), groupingCount, wholeGroup, rec, message, result, fieldNamePrefix);
        }
        return;
    }
    if (groupingCount <= keyPosition) {
        // Entirely in the grouped portion: add fields to groups.
        List<DocumentField> fields = getFields(key, rec, message, fieldNamePrefix);
        for (Map.Entry<Tuple, List<DocumentField>> entry : result.entrySet()) {
            if (TupleHelpers.isPrefix(groupPrefix, entry.getKey())) {
                entry.getValue().addAll(fields);
            }
        }
    // Grouping ends in the middle of this key: break it apart.
    } else if (key instanceof NestingKeyExpression) {
        NestingKeyExpression nesting = (NestingKeyExpression) key;
        final String parentFieldName = nesting.getParent().getFieldName();
        for (Key.Evaluated value : nesting.getParent().evaluateMessage(rec, message)) {
            final Message submessage = (Message) value.toList().get(0);
            getGroupedFields(Collections.singletonList(nesting.getChild()), 0, keyPosition, groupingCount, groupPrefix, rec, submessage, result, fieldNamePrefix == null ? parentFieldName : fieldNamePrefix + "_" + parentFieldName);
        }
    } else if (key instanceof ThenKeyExpression) {
        ThenKeyExpression then = (ThenKeyExpression) key;
        getGroupedFields(then.getChildren(), 0, keyPosition, groupingCount, groupPrefix, rec, message, result, fieldNamePrefix);
    } else {
        throw new RecordCoreException("Cannot split key for document grouping: " + key);
    }
    // Continue with remaining keys.
    getGroupedFields(keys, keyIndex + 1, keyPosition + key.getColumnSize(), groupingCount, groupPrefix, rec, message, result, fieldNamePrefix);
}

Example 15 with ThenKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression in project fdb-record-layer by FoundationDB.

the class RecordQueryPlanner method planOneOfThemWithComparison.

@Nullable
private ScoredPlan planOneOfThemWithComparison(@Nonnull CandidateScan candidateScan, @Nonnull KeyExpression indexExpr, @Nonnull OneOfThemWithComparison oneOfThemWithComparison, @Nullable KeyExpression sort) {
    final Comparisons.Comparison comparison = oneOfThemWithComparison.getComparison();
    final ScanComparisons scanComparisons = ScanComparisons.from(comparison);
    if (scanComparisons == null) {
        final ScoredPlan sortOnlyPlan = planSortOnly(candidateScan, indexExpr, sort);
        if (sortOnlyPlan != null) {
            return new ScoredPlan(0, sortOnlyPlan.plan, Collections.<QueryComponent>singletonList(oneOfThemWithComparison), sortOnlyPlan.createsDuplicates);
        } else {
            return null;
        }
    }
    if (indexExpr instanceof FieldKeyExpression) {
        FieldKeyExpression field = (FieldKeyExpression) indexExpr;
        if (Objects.equals(oneOfThemWithComparison.getFieldName(), field.getFieldName()) && field.getFanType() == FanType.FanOut) {
            if (sort != null) {
                if (sort instanceof FieldKeyExpression) {
                    FieldKeyExpression sortField = (FieldKeyExpression) sort;
                    if (Objects.equals(sortField.getFieldName(), field.getFieldName())) {
                        // everything matches, yay!! Hopefully that comparison can be for tuples
                        return new ScoredPlan(1, valueScan(candidateScan, scanComparisons, true), Collections.<QueryComponent>emptyList(), true);
                    }
                }
            } else {
                return new ScoredPlan(1, valueScan(candidateScan, scanComparisons, false), Collections.<QueryComponent>emptyList(), true);
            }
        }
        return null;
    } else if (indexExpr instanceof ThenKeyExpression) {
        // May need second column to do sort, so handle like And, which does such cases.
        ThenKeyExpression then = (ThenKeyExpression) indexExpr;
        return new AndWithThenPlanner(candidateScan, then, Collections.singletonList(oneOfThemWithComparison), sort).plan();
    } else if (indexExpr instanceof NestingKeyExpression) {
        return null;
    }
    return null;
}