Examples with IndexAggregateFunction com.apple.foundationdb.record.metadata.IndexAggregateFunction used on opensource projects

Example 16 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class FDBRestrictedIndexQueryTest method queryAggregateWithDisabled.

/**
 * Verify that disabled aggregate indexes are not used by the planner.
 * Verify that re-enabling those indexes allows the planner to use them.
 * TODO: Abstract out common code in queryWithWriteOnly, queryWithDisabled, queryAggregateWithWriteOnly and queryAggregateWithDisabled (https://github.com/FoundationDB/fdb-record-layer/issues/4)
 */
@Test
void queryAggregateWithDisabled() throws Exception {
    Index sumIndex = new Index("value3sum", field("num_value_3_indexed").ungrouped(), IndexTypes.SUM);
    Index maxIndex = new Index("value3max", field("num_value_3_indexed").ungrouped(), IndexTypes.MAX_EVER_TUPLE);
    RecordMetaDataHook hook = metaData -> {
        metaData.addIndex("MySimpleRecord", sumIndex);
        metaData.addIndex("MySimpleRecord", maxIndex);
    };
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteAllRecords();
        recordStore.markIndexDisabled("value3sum").join();
        recordStore.markIndexDisabled("value3max").join();
        saveSimpleRecord(1066, 42);
        saveSimpleRecord(1776, 100);
        assertThrowsAggregateFunctionNotSupported(() -> recordStore.evaluateAggregateFunction(Collections.singletonList("MySimpleRecord"), new IndexAggregateFunction(FunctionNames.SUM, sumIndex.getRootExpression(), sumIndex.getName()), TupleRange.ALL, IsolationLevel.SERIALIZABLE).get(), "value3sum.sum(Field { 'num_value_3_indexed' None} group 1)");
        assertThrowsAggregateFunctionNotSupported(() -> recordStore.evaluateAggregateFunction(Collections.singletonList("MySimpleRecord"), new IndexAggregateFunction(FunctionNames.MAX_EVER, maxIndex.getRootExpression(), maxIndex.getName()), TupleRange.ALL, IsolationLevel.SERIALIZABLE).get(), "value3max.max_ever(Field { 'num_value_3_indexed' None} group 1)");
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.uncheckedMarkIndexReadable("value3sum").join();
        recordStore.uncheckedMarkIndexReadable("value3max").join();
        // Unsafe: made readable without building indexes, which is why sum gets wrong answer.
        assertEquals(0L, recordStore.evaluateAggregateFunction(Collections.singletonList("MySimpleRecord"), new IndexAggregateFunction(FunctionNames.SUM, sumIndex.getRootExpression(), sumIndex.getName()), TupleRange.ALL, IsolationLevel.SERIALIZABLE).get().getLong(0));
        assertNull(recordStore.evaluateAggregateFunction(Collections.singletonList("MySimpleRecord"), new IndexAggregateFunction(FunctionNames.MAX_EVER, maxIndex.getRootExpression(), maxIndex.getName()), TupleRange.ALL, IsolationLevel.SERIALIZABLE).get());
        recordStore.rebuildAllIndexes().get();
        assertEquals(142L, recordStore.evaluateAggregateFunction(Collections.singletonList("MySimpleRecord"), new IndexAggregateFunction(FunctionNames.SUM, sumIndex.getRootExpression(), sumIndex.getName()), TupleRange.ALL, IsolationLevel.SERIALIZABLE).get().getLong(0));
        assertEquals(100L, recordStore.evaluateAggregateFunction(Collections.singletonList("MySimpleRecord"), new IndexAggregateFunction(FunctionNames.MAX_EVER, maxIndex.getRootExpression(), maxIndex.getName()), TupleRange.ALL, IsolationLevel.SERIALIZABLE).get().getLong(0));
    }
}

Example 17 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class LeaderboardIndexTest method basicGrouped.

protected void basicGrouped(Leaderboards leaderboards) {
    basicSetup(leaderboards, true);
    try (FDBRecordContext context = openContext()) {
        leaderboards.openRecordStore(context, false);
        TupleRange game_1 = TupleRange.allOf(Tuple.from("game-1"));
        assertEquals(Arrays.asList("patroclus", "hecuba", "achilles", "hector"), leaderboards.scanIndexByRank(game_1).map(leaderboards::getName).asList().join());
        TupleRange top_2 = new TupleRange(Tuple.from("game-1", 0), Tuple.from("game-1", 1), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE);
        assertEquals(Arrays.asList("patroclus", "hecuba"), leaderboards.scanIndexByRank(top_2).map(leaderboards::getName).asList().join());
        TupleRange no_2 = TupleRange.allOf(Tuple.from("game-1", 1));
        assertEquals(Arrays.asList("hecuba"), leaderboards.scanIndexByRank(no_2).map(leaderboards::getName).asList().join());
        TimeWindowScanRange ten_units = new TimeWindowScanRange(TEN_UNITS, 10100, TupleRange.allOf(Tuple.from("game-1")));
        assertEquals(Arrays.asList("achilles", "hector"), leaderboards.scanIndexByTimeWindow(ten_units).map(leaderboards::getName).asList().join());
        assertEquals(Arrays.asList("achilles"), leaderboards.scanIndexByTimeWindowWithLimit(ten_units, 1).map(leaderboards::getName).asList().join());
        TimeWindowScanRange top_2_ten_units = new TimeWindowScanRange(TEN_UNITS, 10102, new TupleRange(Tuple.from("game-1", 0), Tuple.from("game-1", 1), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE));
        assertEquals(Arrays.asList("achilles", "hector"), leaderboards.scanIndexByTimeWindow(top_2_ten_units).map(leaderboards::getName).asList().join());
        TimeWindowScanRange top_2_five_units = new TimeWindowScanRange(FIVE_UNITS, 10100, new TupleRange(Tuple.from("game-1", 0), Tuple.from("game-1", 1), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE));
        assertEquals(Arrays.asList("hector", "achilles"), leaderboards.scanIndexByTimeWindow(top_2_five_units).map(leaderboards::getName).asList().join());
        final FDBStoredRecord<Message> rec1 = leaderboards.findByName("patroclus");
        final FDBStoredRecord<Message> rec2 = leaderboards.findByName("achilles");
        final FDBStoredRecord<Message> rec3 = leaderboards.findByName("hecuba");
        final QueryRecordFunction<Long> rank1 = leaderboards.queryRank();
        assertEquals((Long) 0L, leaderboards.evaluateQueryFunction(rank1, rec1));
        assertEquals((Long) 2L, leaderboards.evaluateQueryFunction(rank1, rec2));
        assertEquals((Long) 1L, leaderboards.evaluateQueryFunction(rank1, rec3));
        final QueryRecordFunction<Long> rank2 = leaderboards.queryTimeWindowRank(TEN_UNITS, 10100);
        assertEquals(null, leaderboards.evaluateQueryFunction(rank2, rec1));
        assertEquals((Long) 0L, leaderboards.evaluateQueryFunction(rank2, rec2));
        assertEquals(null, leaderboards.evaluateQueryFunction(rank2, rec3));
        final QueryRecordFunction<Tuple> entry1 = leaderboards.queryTimeWindowRankAndEntry(TimeWindowLeaderboard.ALL_TIME_LEADERBOARD_TYPE, -1);
        assertEquals(Tuple.from(0, 1000, 11001, 111), leaderboards.evaluateQueryFunction(entry1, rec1));
        assertEquals(Tuple.from(2, 300, 10201, 668), leaderboards.evaluateQueryFunction(entry1, rec2));
        assertEquals(Tuple.from(1, 750, 11201, 888), leaderboards.evaluateQueryFunction(entry1, rec3));
        final QueryRecordFunction<Tuple> entry2 = leaderboards.queryTimeWindowRankAndEntry(TEN_UNITS, 10100);
        assertEquals(null, leaderboards.evaluateQueryFunction(entry2, rec1));
        assertEquals(Tuple.from(0, 200, 10105, 667), leaderboards.evaluateQueryFunction(entry2, rec2));
        assertEquals(null, leaderboards.evaluateQueryFunction(entry2, rec3));
        final IndexAggregateFunction count1 = leaderboards.timeWindowCount(TimeWindowLeaderboard.ALL_TIME_LEADERBOARD_TYPE, -1);
        assertEquals((Long) 4L, leaderboards.evaluateAggregateFunction(count1, Tuple.from("game-1")).get(0));
        assertEquals((Long) 1L, leaderboards.evaluateAggregateFunction(count1, Tuple.from("game-2")).get(0));
        final IndexAggregateFunction count2 = leaderboards.timeWindowCount(TEN_UNITS, 10100);
        assertEquals((Long) 2L, leaderboards.evaluateAggregateFunction(count2, Tuple.from("game-1")).get(0));
    }
}

Example 18 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method sumBoundIndex.

@Test
public void sumBoundIndex() throws Exception {
    final FieldKeyExpression recno = field("rec_no");
    final GroupingKeyExpression byKey = recno.groupBy(field("num_value_3_indexed"));
    final RecordMetaDataHook hook = md -> md.addUniversalIndex(new Index("sum", byKey, IndexTypes.SUM));
    final IndexAggregateFunction subtotal = new IndexAggregateFunction(FunctionNames.SUM, byKey, null);
    final List<String> allTypes = Collections.emptyList();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        final Optional<IndexAggregateFunction> boundSubtotal = IndexFunctionHelper.bindAggregateFunction(recordStore, subtotal, allTypes, IndexQueryabilityFilter.DEFAULT);
        assertTrue(boundSubtotal.isPresent(), "should find a suitable index");
        assertEquals("sum", boundSubtotal.get().getIndex());
        final Optional<IndexAggregateGroupKeys> keyFunction = IndexAggregateGroupKeys.conditionsToGroupKeys(subtotal, Query.field("num_value_3_indexed").equalsValue(1));
        assertTrue(keyFunction.isPresent(), "should match conditions");
        final Key.Evaluated keys = keyFunction.get().getGroupKeys(recordStore, EvaluationContext.EMPTY);
        assertEquals(Key.Evaluated.scalar(1), keys);
        assertEquals((99 * 100) / (2 * 5) - 20, recordStore.evaluateAggregateFunction(allTypes, subtotal, keys, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals((99 * 100) / (2 * 5) - 20, recordStore.evaluateAggregateFunction(allTypes, boundSubtotal.get(), keys, IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}

Example 19 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxTupleUngrouped.

@Test
public void minMaxTupleUngrouped() throws Exception {
    final FieldKeyExpression fieldKey = field("num_value_3_indexed");
    final GroupingKeyExpression indexKey = fieldKey.ungrouped();
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("min", indexKey, IndexTypes.MIN_EVER_TUPLE));
        md.addIndex(type, new Index("max", indexKey, IndexTypes.MAX_EVER_TUPLE));
    };
    final IndexAggregateFunction min = new IndexAggregateFunction(FunctionNames.MIN_EVER, fieldKey, null);
    final IndexAggregateFunction max = new IndexAggregateFunction(FunctionNames.MAX_EVER, fieldKey, null);
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertNull(recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setNumValue3Indexed(i * 10);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(Tuple.from(0), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(990), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        commit(context);
    }
}

Example 20 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxValueNegative.

@Test
public void minMaxValueNegative() throws Exception {
    final FieldKeyExpression strValue = field("str_value_indexed");
    final FieldKeyExpression numValue2 = field("num_value_2");
    final FieldKeyExpression numValue3 = field("num_value_3_indexed");
    final ThenKeyExpression compound = concat(strValue, numValue2, numValue3);
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("compound", compound, IndexTypes.VALUE));
    };
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        final IndexAggregateFunction minValue2 = new IndexAggregateFunction(FunctionNames.MIN, numValue2, null);
        assertThrows(RecordCoreException.class, () -> {
            recordStore.evaluateAggregateFunction(types, minValue2, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join();
        });
        final IndexAggregateFunction minValue3GroupedIncorrectly = new IndexAggregateFunction(FunctionNames.MIN, numValue3.groupBy(numValue2, strValue), null);
        assertThrows(RecordCoreException.class, () -> {
            recordStore.evaluateAggregateFunction(types, minValue3GroupedIncorrectly, Key.Evaluated.concatenate(1, "foo"), IsolationLevel.SNAPSHOT).join();
        });
        final IndexAggregateFunction minValue3GroupedTooMany = new IndexAggregateFunction(FunctionNames.MIN, concat(numValue3, numValue2).groupBy(strValue), null);
        assertThrows(RecordCoreException.class, () -> {
            recordStore.evaluateAggregateFunction(types, minValue3GroupedTooMany, Key.Evaluated.scalar("foo"), IsolationLevel.SNAPSHOT).join();
        });
        commit(context);
    }
}