Examples with IndexAggregateFunction com.apple.foundationdb.record.metadata.IndexAggregateFunction used on opensource projects

Example 6 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method countValueIndex.

@Test
public void countValueIndex() throws Exception {
    final FieldKeyExpression numValue3 = field("num_value_3_indexed");
    final GroupingKeyExpression byKey = numValue3.groupBy(field("str_value_indexed"));
    final RecordMetaDataHook hook = md -> md.addIndex("MySimpleRecord", new Index("count_num_3", byKey, IndexTypes.COUNT_NOT_NULL));
    final List<String> types = Collections.singletonList("MySimpleRecord");
    final IndexAggregateFunction perKey = new IndexAggregateFunction(FunctionNames.COUNT_NOT_NULL, byKey, null);
    final IndexAggregateFunction total = new IndexAggregateFunction(FunctionNames.COUNT_NOT_NULL, numValue3, null);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            if (i % 5 == 0) {
                recBuilder.clearNumValue3Indexed();
            } else {
                recBuilder.setNumValue3Indexed(i + 1000);
            }
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(80, recordStore.evaluateAggregateFunction(types, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(40, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteRecord(Tuple.from(8));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(79, recordStore.evaluateAggregateFunction(types, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(39, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}

Example 7 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method countClearWhenZero.

@ParameterizedTest
@BooleanSource
public void countClearWhenZero(boolean clearWhenZero) throws Exception {
    final GroupingKeyExpression byKey = new GroupingKeyExpression(field("str_value_indexed"), 0);
    final RecordMetaDataHook hook = md -> md.addIndex("MySimpleRecord", new Index("count_by_str", byKey, IndexTypes.COUNT, ImmutableMap.of(IndexOptions.CLEAR_WHEN_ZERO, Boolean.toString(clearWhenZero))));
    final List<String> types = Collections.singletonList("MySimpleRecord");
    final IndexAggregateFunction perKey = new IndexAggregateFunction(FunctionNames.COUNT, byKey, null);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 10; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(5, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(5, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("odd"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(ImmutableMap.of("even", 5L, "odd", 5L), recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("count_by_str"), IndexScanType.BY_GROUP, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(i -> Pair.of(i.getKey().get(0), i.getValue().get(0))).asList().join().stream().collect(Collectors.toMap(Pair::getLeft, Pair::getRight)));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 10; i += 2) {
            recordStore.deleteRecord(Tuple.from(i));
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(0, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(5, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("odd"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(clearWhenZero ? ImmutableMap.of("odd", 5L) : ImmutableMap.of("even", 0L, "odd", 5L), recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("count_by_str"), IndexScanType.BY_GROUP, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(i -> Pair.of(i.getKey().get(0), i.getValue().get(0))).asList().join().stream().collect(Collectors.toMap(Pair::getLeft, Pair::getRight)));
        commit(context);
    }
}

Example 8 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class RankIndexTest method rankPlusMatchingNonRankIndex.

@Test
public void rankPlusMatchingNonRankIndex() throws Exception {
    RecordMetaDataHook hook = md -> md.addIndex("BasicRankedRecord", new Index("AaaSumIndex", Key.Expressions.field("score").ungrouped(), IndexTypes.SUM));
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        // Ordinarily the stable order is the persistent form; force new index to the front.
        recordStore.getRecordMetaData().getRecordType("BasicRankedRecord").getIndexes().sort(Comparator.comparing(Index::getName));
        recordStore.rebuildIndex(recordStore.getRecordMetaData().getIndex("AaaSumIndex")).join();
        commit(context);
    }
    // New index should not interfere with score_for_rank choice.
    RecordQuery query = RecordQuery.newBuilder().setRecordType("BasicRankedRecord").setFilter(Query.and(Query.field("gender").equalsValue("F"), Query.rank(Key.Expressions.field("score").ungrouped()).equalsValue(2L))).build();
    RecordQueryPlan plan = planner.plan(query);
    assertEquals("Index(rank_by_gender [EQUALS F, EQUALS $__rank_0])" + " WHERE __rank_0 = BasicRankedRecord$score.score_for_rank(2)", plan.toString());
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        List<String> names = recordStore.executeQuery(plan).map(rec -> TestRecordsRankProto.BasicRankedRecord.newBuilder().mergeFrom(rec.getRecord()).getName()).asList().join();
        assertEquals(Arrays.asList("helen", "penelope"), names);
        assertEquals(875L, recordStore.evaluateAggregateFunction(Collections.singletonList("BasicRankedRecord"), new IndexAggregateFunction(FunctionNames.SUM, Key.Expressions.field("score"), null), Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}

Example 9 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxTupleRepeatConcatenated.

@Test
public void minMaxTupleRepeatConcatenated() throws Exception {
    final FieldKeyExpression fieldKey = field("repeater", FanType.Concatenate);
    final GroupingKeyExpression indexKey = fieldKey.ungrouped();
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("min", indexKey, IndexTypes.MIN_EVER_TUPLE));
        md.addIndex(type, new Index("max", indexKey, IndexTypes.MAX_EVER_TUPLE));
    };
    final IndexAggregateFunction min = new IndexAggregateFunction(FunctionNames.MIN_EVER, indexKey, null);
    final IndexAggregateFunction max = new IndexAggregateFunction(FunctionNames.MAX_EVER, indexKey, null);
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertNull(recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(1);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        recBuilder.addRepeater(1);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from(1L)), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        recBuilder.addRepeater(1);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from(1L, 1L)), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        recBuilder.clearRepeater();
        recBuilder.addRepeater(2);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from(2L)), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        commit(context);
    }
}

Example 10 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method testOnlineIndexerBuilderWriteLimitBytes.

@Test
public void testOnlineIndexerBuilderWriteLimitBytes() throws Exception {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("newIndex", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    final FDBStoreTimer timer = new FDBStoreTimer();
    try (FDBRecordContext context = openContext()) {
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        // Build in this transaction.
        try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setRecordStore(recordStore).setTimer(timer).setIndex("newIndex").setLimit(100000).setMaxWriteLimitBytes(1).build()) {
            // this call will "flatten" the staccato iterations to a whole range. Testing compatibility.
            indexer.rebuildIndex(recordStore);
        }
        recordStore.markIndexReadable("newIndex").join();
        assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
        assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
        assertEquals(199, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
        // last item
        assertEquals(1, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        assertTrue(recordStore.isIndexReadable("newIndex"));
        recordStore.clearAndMarkIndexWriteOnly("newIndex").join();
        context.commit();
    }
    timer.reset();
    try (FDBRecordContext context = openContext()) {
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        // Build in this transaction.
        try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setRecordStore(recordStore).setTimer(timer).setIndex("newIndex").setLimit(100000).setMaxWriteLimitBytes(1).build()) {
            Key.Evaluated key = indexer.buildUnbuiltRange(Key.Evaluated.scalar(0L), Key.Evaluated.scalar(25L)).join();
            assertEquals(1, key.getLong(0));
            assertEquals(1, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
            assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
        }
        recordStore.clearAndMarkIndexWriteOnly("newIndex").join();
        context.commit();
    }
    timer.reset();
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTimer(timer).setIndex(index).setLimit(100000).setMaxWriteLimitBytes(1).setRecordsPerSecond(OnlineIndexer.UNLIMITED).build()) {
        indexer.buildIndex();
    }
    assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    // this includes two endpoints + one range = total of 3 terminations by count
    // - note that (last, null] endpoint is en empty range
    assertEquals(3, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
    // this is the range between the endpoints - 199 items in (first, last] interval
    assertEquals(198, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
}