use of com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method minMaxIndex.
@ParameterizedTest(name = "minMaxIndex({0})")
@EnumSource(MinMaxIndexTypes.class)
public void minMaxIndex(MinMaxIndexTypes indexTypes) throws Exception {
final FieldKeyExpression recno = field("rec_no");
final GroupingKeyExpression byKey = recno.groupBy(field("num_value_3_indexed"));
final RecordMetaDataHook hook = md -> {
RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
md.addIndex(type, new Index("min", byKey, indexTypes.min()));
md.addIndex(type, new Index("max", byKey, indexTypes.max()));
};
final IndexAggregateFunction minOverall = new IndexAggregateFunction(FunctionNames.MIN_EVER, recno, null);
final IndexAggregateFunction maxOverall = new IndexAggregateFunction(FunctionNames.MAX_EVER, recno, null);
final IndexAggregateFunction minByKey = new IndexAggregateFunction(FunctionNames.MIN_EVER, byKey, null);
final IndexAggregateFunction maxByKey = new IndexAggregateFunction(FunctionNames.MAX_EVER, byKey, null);
List<String> types = Collections.singletonList("MySimpleRecord");
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
assertNull(recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
for (int i = 0; i < 100; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(i);
recBuilder.setNumValue3Indexed(i % 5);
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
assertEquals(0, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(99, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(1, recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(96, recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
commit(context);
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.deleteRecord(Tuple.from(0));
recordStore.deleteRecord(Tuple.from(99));
assertEquals(0, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(99, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
commit(context);
}
// verify that negatives do not appear in min/max
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(-1);
recBuilder.setNumValue3Indexed(1);
recordStore.saveRecord(recBuilder.build());
if (!indexTypes.shouldAllowNegative()) {
fail("should have thrown exception");
}
} catch (RecordCoreException e) {
if (indexTypes.shouldAllowNegative()) {
throw e;
}
assertEquals(e.getMessage(), "Attempted update of MAX_EVER_LONG or MIN_EVER_LONG index with negative value");
}
}
use of com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression in project fdb-record-layer by FoundationDB.
the class LuceneIndexExpressions method getFieldsRecursively.
@SuppressWarnings("squid:S3776")
public static <T extends RecordSource<T>> void getFieldsRecursively(@Nonnull KeyExpression expression, @Nonnull T source, @Nonnull DocumentDestination<T> destination, @Nullable String fieldNamePrefix, int keyIndex, int groupingCount, @Nonnull List<Integer> overriddenKeyRanges) {
if (expression instanceof ThenKeyExpression) {
int count = 0;
for (KeyExpression child : ((ThenKeyExpression) expression).getChildren()) {
getFieldsRecursively(child, source, destination, fieldNamePrefix, keyIndex + count, groupingCount, overriddenKeyRanges);
count += child.getColumnSize();
}
return;
}
String fieldNameSuffix = null;
boolean suffixOverride = false;
if (expression instanceof LuceneFunctionKeyExpression.LuceneFieldName) {
LuceneFunctionKeyExpression.LuceneFieldName fieldNameExpression = (LuceneFunctionKeyExpression.LuceneFieldName) expression;
KeyExpression nameExpression = fieldNameExpression.getNameExpression();
if (nameExpression instanceof LiteralKeyExpression) {
fieldNameSuffix = (String) ((LiteralKeyExpression<?>) nameExpression).getValue();
} else if (nameExpression instanceof FieldKeyExpression) {
Iterator<Object> names = source.getValues((FieldKeyExpression) nameExpression).iterator();
if (names.hasNext()) {
fieldNameSuffix = (String) names.next();
if (names.hasNext()) {
throw new RecordCoreException("Lucene field name override should evaluate to single value");
}
}
} else {
throw new RecordCoreException("Lucene field name override should be a literal or a field");
}
suffixOverride = true;
expression = fieldNameExpression.getNamedExpression();
}
if (expression instanceof NestingKeyExpression) {
NestingKeyExpression nestingExpression = (NestingKeyExpression) expression;
FieldKeyExpression parentExpression = nestingExpression.getParent();
KeyExpression child = nestingExpression.getChild();
if (!suffixOverride) {
fieldNameSuffix = parentExpression.getFieldName();
} else {
addOverriddenKeyRange(overriddenKeyRanges, fieldNamePrefix, fieldNameSuffix);
}
String fieldName = appendFieldName(fieldNamePrefix, fieldNameSuffix);
for (T subsource : source.getChildren(parentExpression)) {
getFieldsRecursively(child, subsource, destination, fieldName, keyIndex, groupingCount, overriddenKeyRanges);
}
if (suffixOverride) {
// Remove the last 2 numbers added above
removedLastOverriddenKeyRange(overriddenKeyRanges);
}
return;
}
boolean fieldStored = false;
boolean fieldText = false;
while (true) {
if (expression instanceof LuceneFunctionKeyExpression.LuceneStored) {
LuceneFunctionKeyExpression.LuceneStored storedExpression = (LuceneFunctionKeyExpression.LuceneStored) expression;
fieldStored = true;
expression = storedExpression.getStoredExpression();
} else if (expression instanceof LuceneFunctionKeyExpression.LuceneText) {
LuceneFunctionKeyExpression.LuceneText textExpression = (LuceneFunctionKeyExpression.LuceneText) expression;
fieldText = true;
expression = textExpression.getFieldExpression();
} else {
// TODO: More text options.
break;
}
}
if (expression instanceof FieldKeyExpression) {
FieldKeyExpression fieldExpression = (FieldKeyExpression) expression;
if (!suffixOverride) {
fieldNameSuffix = fieldExpression.getFieldName();
} else {
addOverriddenKeyRange(overriddenKeyRanges, fieldNamePrefix, fieldNameSuffix);
}
String fieldName = appendFieldName(fieldNamePrefix, fieldNameSuffix);
if (fieldName == null) {
fieldName = "_";
}
Descriptors.Descriptor recordDescriptor = source.getDescriptor();
Descriptors.FieldDescriptor fieldDescriptor = recordDescriptor.findFieldByName(fieldExpression.getFieldName());
DocumentFieldType fieldType;
if (fieldText) {
switch(fieldDescriptor.getJavaType()) {
case STRING:
fieldType = DocumentFieldType.TEXT;
break;
default:
throw new RecordCoreException("Unknown Lucene text field type");
}
} else {
switch(fieldDescriptor.getJavaType()) {
case STRING:
fieldType = DocumentFieldType.STRING;
break;
case INT:
fieldType = DocumentFieldType.INT;
break;
case LONG:
fieldType = DocumentFieldType.LONG;
break;
case DOUBLE:
fieldType = DocumentFieldType.DOUBLE;
break;
case BOOLEAN:
fieldType = DocumentFieldType.BOOLEAN;
break;
default:
throw new RecordCoreException("Unknown Lucene field type");
}
}
for (Object value : source.getValues(fieldExpression)) {
destination.addField(source, fieldName, value, fieldType, fieldStored, overriddenKeyRanges, keyIndex < groupingCount ? keyIndex : -1);
}
if (suffixOverride) {
// Remove the last 2 numbers added above
removedLastOverriddenKeyRange(overriddenKeyRanges);
}
return;
}
throw new RecordCoreException("Unknown Lucene field key expression");
}
use of com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression in project fdb-record-layer by FoundationDB.
the class RecordMetaDataBuilder method protoFieldOptions.
@SuppressWarnings("deprecation")
private void protoFieldOptions(RecordTypeBuilder recordType, Descriptors.FieldDescriptor fieldDescriptor, RecordMetaDataOptionsProto.FieldOptions fieldOptions) {
Descriptors.Descriptor descriptor = recordType.getDescriptor();
if (fieldOptions.hasIndex() || fieldOptions.hasIndexed()) {
String type;
Map<String, String> options;
if (fieldOptions.hasIndex()) {
RecordMetaDataOptionsProto.FieldOptions.IndexOption indexOption = fieldOptions.getIndex();
type = indexOption.getType();
options = Index.buildOptions(indexOption.getOptionsList(), indexOption.getUnique());
} else {
type = Index.indexTypeToType(fieldOptions.getIndexed());
options = Index.indexTypeToOptions(fieldOptions.getIndexed());
}
final FieldKeyExpression field = Key.Expressions.fromDescriptor(fieldDescriptor);
final KeyExpression expr;
if (type.equals(IndexTypes.RANK)) {
expr = field.ungrouped();
} else {
expr = field;
}
final Index index = new Index(descriptor.getName() + "$" + fieldDescriptor.getName(), expr, Index.EMPTY_VALUE, type, options);
addIndex(recordType, index);
} else if (fieldOptions.getPrimaryKey()) {
if (recordType.getPrimaryKey() != null) {
throw new MetaDataException("Only one primary key per record type is allowed have: " + recordType.getPrimaryKey() + "; adding on " + fieldDescriptor.getName());
} else {
if (fieldDescriptor.isRepeated()) {
// TODO maybe default to concatenate for this.
throw new MetaDataException("Primary key cannot be set on a repeated field");
} else {
recordType.setPrimaryKey(Key.Expressions.fromDescriptor(fieldDescriptor));
}
}
}
}
use of com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression in project fdb-record-layer by FoundationDB.
the class RecordQueryPlanner method planOneOfThemWithComparison.
@Nullable
private ScoredPlan planOneOfThemWithComparison(@Nonnull CandidateScan candidateScan, @Nonnull KeyExpression indexExpr, @Nonnull OneOfThemWithComparison oneOfThemWithComparison, @Nullable KeyExpression sort) {
final Comparisons.Comparison comparison = oneOfThemWithComparison.getComparison();
final ScanComparisons scanComparisons = ScanComparisons.from(comparison);
if (scanComparisons == null) {
final ScoredPlan sortOnlyPlan = planSortOnly(candidateScan, indexExpr, sort);
if (sortOnlyPlan != null) {
return new ScoredPlan(0, sortOnlyPlan.plan, Collections.<QueryComponent>singletonList(oneOfThemWithComparison), sortOnlyPlan.createsDuplicates);
} else {
return null;
}
}
if (indexExpr instanceof FieldKeyExpression) {
FieldKeyExpression field = (FieldKeyExpression) indexExpr;
if (Objects.equals(oneOfThemWithComparison.getFieldName(), field.getFieldName()) && field.getFanType() == FanType.FanOut) {
if (sort != null) {
if (sort instanceof FieldKeyExpression) {
FieldKeyExpression sortField = (FieldKeyExpression) sort;
if (Objects.equals(sortField.getFieldName(), field.getFieldName())) {
// everything matches, yay!! Hopefully that comparison can be for tuples
return new ScoredPlan(1, valueScan(candidateScan, scanComparisons, true), Collections.<QueryComponent>emptyList(), true);
}
}
} else {
return new ScoredPlan(1, valueScan(candidateScan, scanComparisons, false), Collections.<QueryComponent>emptyList(), true);
}
}
return null;
} else if (indexExpr instanceof ThenKeyExpression) {
// May need second column to do sort, so handle like And, which does such cases.
ThenKeyExpression then = (ThenKeyExpression) indexExpr;
return new AndWithThenPlanner(candidateScan, then, Collections.singletonList(oneOfThemWithComparison), sort).plan();
} else if (indexExpr instanceof NestingKeyExpression) {
return null;
}
return null;
}
use of com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression in project fdb-record-layer by FoundationDB.
the class RecordQueryPlanner method planOneOfThemWithComponent.
@Nullable
private ScoredPlan planOneOfThemWithComponent(@Nonnull CandidateScan candidateScan, @Nonnull KeyExpression indexExpr, @Nonnull OneOfThemWithComponent filter, @Nullable KeyExpression sort) {
if (indexExpr instanceof FieldKeyExpression) {
return null;
} else if (indexExpr instanceof ThenKeyExpression) {
ThenKeyExpression then = (ThenKeyExpression) indexExpr;
return planOneOfThemWithComponent(candidateScan, then.getChildren().get(0), filter, sort);
} else if (indexExpr instanceof NestingKeyExpression) {
NestingKeyExpression indexNesting = (NestingKeyExpression) indexExpr;
ScoredPlan plan = null;
if (sort == null) {
plan = planNesting(candidateScan, indexNesting, filter, null);
} else if (sort instanceof FieldKeyExpression) {
plan = null;
} else if (sort instanceof ThenKeyExpression) {
plan = null;
} else if (sort instanceof NestingKeyExpression) {
NestingKeyExpression sortNesting = (NestingKeyExpression) sort;
plan = planNesting(candidateScan, indexNesting, filter, sortNesting);
}
if (plan != null) {
List<QueryComponent> unsatisfied;
if (!plan.unsatisfiedFilters.isEmpty()) {
unsatisfied = Collections.singletonList(filter);
} else {
unsatisfied = Collections.emptyList();
}
// Right now it marks the whole nesting as unsatisfied, in theory there could be plans that handle that
plan = new ScoredPlan(plan.score, plan.plan, unsatisfied, true);
}
return plan;
}
return null;
}
Aggregations