use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class IndexFunctionHelperTest method filterIndexForBindAggregateFunction.
@Test
void filterIndexForBindAggregateFunction() {
final GroupingKeyExpression group = concat(field("str_value_indexed"), field("num_value_2")).group(1);
RecordMetaDataHook hook = metaData -> {
metaData.addIndex("MySimpleRecord", new Index("filtered_sum_value2", group, Index.EMPTY_VALUE, IndexTypes.SUM, Map.of()));
};
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.deleteAllRecords();
final IndexAggregateFunction indexAggregateFunction = new IndexAggregateFunction("sum", group, null);
final Optional<IndexAggregateFunction> expected = Optional.of(new IndexAggregateFunction("sum", group, "filtered_sum_value2"));
assertEquals(expected, IndexFunctionHelper.bindAggregateFunction(recordStore, indexAggregateFunction, List.of("MySimpleRecord"), IndexQueryabilityFilter.TRUE));
assertEquals(Optional.empty(), IndexFunctionHelper.bindAggregateFunction(recordStore, indexAggregateFunction, List.of("MySimpleRecord"), IndexQueryabilityFilter.FALSE));
commit(context);
}
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class OnlineIndexerIndexFromIndexTest method testIndexFromIndexNoFallbackNonValueSrc.
@Test
public void testIndexFromIndexNoFallbackNonValueSrc() {
// Let srcIndex be a non-VALUE index
final FDBStoreTimer timer = new FDBStoreTimer();
final long numRecords = 3;
Index srcIndex = new Index("src_index", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
Index tgtIndex = new Index("tgt_index", field("num_value_3_indexed"), IndexTypes.VALUE);
FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(srcIndex, tgtIndex);
openSimpleMetaData();
populateData(numRecords);
openSimpleMetaData(hook);
buildSrcIndex(srcIndex);
openSimpleMetaData(hook);
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().build()).setTimer(timer).build()) {
IndexingByIndex.ValidationException e = assertThrows(IndexingByIndex.ValidationException.class, indexBuilder::buildIndex);
assertTrue(e.getMessage().contains("source index is not a VALUE index"));
}
assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class RankIndexTest method repeatedRank.
public void repeatedRank(List<TestRecordsRankProto.RepeatedRankedRecord> records) throws Exception {
fdb = FDBDatabaseFactory.instance().getDatabase();
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
// Undo loadRecords().
recordStore.deleteAllRecords();
for (TestRecordsRankProto.RepeatedRankedRecord record : records) {
recordStore.saveRecord(record);
}
commit(context);
}
final List<Pair<Integer, String>> recordsSortedByRankWithDuplicates = records.stream().flatMap(record -> record.getScoreList().stream().map(score -> Pair.of(score, record.getName()))).sorted(Comparator.comparing(Pair::getLeft)).collect(Collectors.toList());
List<Set<String>> rankWithTies = new ArrayList<>();
Integer lastScore = null;
for (Pair<Integer, String> recordsSortedByRankWithDuplicate : recordsSortedByRankWithDuplicates) {
int score = recordsSortedByRankWithDuplicate.getLeft();
final String name = recordsSortedByRankWithDuplicate.getRight();
if (lastScore == null || !lastScore.equals(score)) {
// A set as the same record can have the same score multiple times, but each unique score,
// per record, will only be counted once
Set<String> tie = new HashSet<>();
tie.add(name);
rankWithTies.add(tie);
} else {
rankWithTies.get(rankWithTies.size() - 1).add(name);
}
lastScore = score;
}
GroupingKeyExpression expr = Key.Expressions.field("score", KeyExpression.FanType.FanOut).ungrouped();
RecordQuery.Builder builder = RecordQuery.newBuilder().setRecordType("RepeatedRankedRecord").setFilter(Query.rank(expr).withParameterComparison(Comparisons.Type.EQUALS, "RANK_VALUE"));
RecordQuery query = builder.setRemoveDuplicates(false).build();
RecordQueryPlan plan = planner.plan(query);
assertAll(IntStream.range(0, rankWithTies.size()).mapToObj(i -> () -> {
Set<String> tie = rankWithTies.get(i);
try (FDBRecordContext context = openContext()) {
try {
openRecordStore(context);
} catch (Exception e) {
Assertions.fail(e);
}
final List<String> actualRecords = plan.execute(recordStore, EvaluationContext.forBinding("RANK_VALUE", i)).map(rec -> TestRecordsRankProto.RepeatedRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().join();
assertThat("For Rank " + i, actualRecords, containsInAnyOrder(tie.toArray()));
}
}));
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class RankIndexTest method repeatedRankQuery.
@Test
public void repeatedRankQuery() throws Exception {
fdb = FDBDatabaseFactory.instance().getDatabase();
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
// Undo loadRecords().
recordStore.deleteAllRecords();
TestRecordsRankProto.RepeatedRankedRecord.Builder rec = TestRecordsRankProto.RepeatedRankedRecord.newBuilder();
rec.setName("patroclus").addScore(-5).addScore(5).addScore(-10).addScore(-11).addScore(-8);
recordStore.saveRecord(rec.build());
rec.clear();
rec.setName("achilles").addScore(-14).addScore(5).addScore(9).addScore(-8).addScore(-1).addScore(-16);
recordStore.saveRecord(rec.build());
rec.clear();
rec.setName("hector").addScore(-5).addScore(5).addScore(-3).addScore(-2).addScore(0).addScore(10);
recordStore.saveRecord(rec.build());
commit(context);
}
// Reordered by score:
// [ -16 (Achilles), -14 (Achilles), -11 (Patroclus), -10 (Patroclus), -8 (Achilles, Patroclus), -5 (Hector, Patroclus),
// -3 (Hector), -2 (Hector), -1 (Achilles), 0 (Hector), 5 (Achilles, Hector, Patroclus), 9 (Achilles), 10 (Hector)]
GroupingKeyExpression expr = Key.Expressions.field("score", KeyExpression.FanType.FanOut).ungrouped();
RecordQuery.Builder builder = RecordQuery.newBuilder().setRecordType("RepeatedRankedRecord").setFilter(Query.and(Query.rank(expr).greaterThanOrEquals(4L), Query.rank(expr).lessThanOrEquals(10L)));
RecordQuery query = builder.setRemoveDuplicates(false).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(score_by_repeated_field [[4],[10]] BY_RANK)", plan.toString());
List<String> res;
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
res = recordStore.executeQuery(plan).map(rec -> TestRecordsRankProto.RepeatedRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().join();
}
assertEquals(Arrays.asList("achilles", "patroclus", "hector", "patroclus", "hector", "hector", "achilles", "hector", "achilles", "hector", "patroclus"), res);
query = builder.setRemoveDuplicates(true).build();
plan = planner.plan(query);
assertEquals("Index(score_by_repeated_field [[4],[10]] BY_RANK) | UnorderedPrimaryKeyDistinct()", plan.toString());
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
res = recordStore.executeQuery(plan).map(rec -> TestRecordsRankProto.RepeatedRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().join();
}
assertEquals(Arrays.asList("achilles", "patroclus", "hector"), res);
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class BitmapValueIndexTest method nestedAndQuery.
@Test
void nestedAndQuery() {
final KeyExpression num_by_str = field("nested").nest(field("entry", FanOut).nest(concatenateFields("str_value", "num_value")));
final GroupingKeyExpression nested_num_by_str = concat(field("num_value_1"), num_by_str).group(1);
final KeyExpression nested_num_by_str_num2 = concat(field("num_value_1"), field("num_value_2"), num_by_str).group(1);
final KeyExpression nested_num_by_str_num3 = concat(field("num_value_1"), field("num_value_3"), num_by_str).group(1);
final RecordMetaDataHook nested_rec_no_by_str_nums_hook = metadata -> {
final RecordTypeBuilder recordType = metadata.getRecordType("MyNestedRecord");
metadata.addIndex(recordType, new Index("nested_num_by_str_num2", nested_num_by_str_num2, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
metadata.addIndex(recordType, new Index("nested_num_by_str_num3", nested_num_by_str_num3, IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
};
final IndexAggregateFunctionCall bitmap_value_nested_num_by_str = new IndexAggregateFunctionCall(FunctionNames.BITMAP_VALUE, nested_num_by_str);
try (FDBRecordContext context = openContext()) {
createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
for (int recNo = 100; recNo < 200; recNo++) {
recordStore.saveRecord(TestRecordsBitmapProto.MyNestedRecord.newBuilder().setRecNo(recNo).setNumValue1(1).setNested(TestRecordsBitmapProto.MyNestedRecord.Nested.newBuilder().addEntry(TestRecordsBitmapProto.MyNestedRecord.Nested.Entry.newBuilder().setStrValue((recNo & 1) == 1 ? "odd" : "even").setNumValue(recNo + 1000))).setNumValue2(recNo % 7).setNumValue3(recNo % 5).build());
}
commit(context);
}
try (FDBRecordContext context = openContext()) {
createOrOpenRecordStore(context, metaData(nested_rec_no_by_str_nums_hook));
setupPlanner(null);
final RecordQuery recordQuery = RecordQuery.newBuilder().setRecordType("MyNestedRecord").setFilter(Query.and(Query.field("num_value_1").equalsValue(1), Query.field("nested").matches(Query.field("entry").oneOfThem().matches(Query.field("str_value").equalsValue("odd"))), Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4))).setRequiredResults(Collections.singletonList(field("nested").nest(field("entry", FanOut).nest("num_value")))).build();
final RecordQueryPlan queryPlan = ComposedBitmapIndexAggregate.tryPlan((RecordQueryPlanner) planner, recordQuery, bitmap_value_nested_num_by_str, IndexQueryabilityFilter.DEFAULT).orElseGet(() -> fail("Cannot plan query"));
assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 3, odd],[1, 3, odd]]"))))), coveringIndexScan(indexScan(allOf(indexName("nested_num_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[1, 4, odd],[1, 4, odd]]"))))))));
assertEquals(1000204717, queryPlan.planHash());
assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && (i % 5) == 4).map(i -> i + 1000).collect(Collectors.toList())));
}
}
Aggregations