Examples with IndexTypes com.apple.foundationdb.record.metadata.IndexTypes used on opensource projects

Example 6 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildEndpointIdempotency.

@Test
public void buildEndpointIdempotency() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 10).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit the change.
        }
        return ret;
    };
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        recordStore.markIndexWriteOnly(index).join();
        context.commit();
    }
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        final RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(index));
        // Build the endpoints
        TupleRange range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(10L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(0L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(9L).pack(), null).join());
        List<Range> middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(0L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(9L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Make sure running this again doesn't change anything.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(10L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(0L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(9L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(0L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(9L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Remove the first and last records.
        try (FDBRecordContext context = openContext()) {
            recordStore.deleteRecord(Tuple.from(0L));
            recordStore.deleteRecord(Tuple.from(9L));
            context.commit();
        }
        assertEquals(Tuple.from(0L), getAggregate.get());
        // Rerun endpoints with new data.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(1L), range.getLow());
        assertEquals(Tuple.from(8L), range.getHigh());
        assertEquals(Tuple.from(9L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Run it again to show that nothing has happened.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(1L), range.getLow());
        assertEquals(Tuple.from(8L), range.getHigh());
        assertEquals(Tuple.from(9L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Add back the previous first and last records.
        try (FDBRecordContext context = openContext()) {
            recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(0L).setNumValue2(1).build());
            recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(9L).setNumValue2(10).build());
            context.commit();
        }
        assertEquals(Tuple.from(20L), getAggregate.get());
        // Rerun endpoints with new data.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(20L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Run it again to show that nothing has happened.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(20L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Straight up build the whole index.
        indexBuilder.buildIndex(false);
        assertEquals(Tuple.from(55L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database()).join());
    }
}

Example 7 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxOptional.

@ParameterizedTest(name = "minMaxLongOptional({0})")
@EnumSource(MinMaxIndexTypes.class)
public void minMaxOptional(MinMaxIndexTypes indexTypes) throws Exception {
    final KeyExpression key = field("num_value_3_indexed").ungrouped();
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("min", key, indexTypes.min()));
        md.addIndex(type, new Index("max", key, indexTypes.max()));
    };
    final IndexAggregateFunction minOverall = new IndexAggregateFunction(FunctionNames.MIN_EVER, key, null);
    final IndexAggregateFunction maxOverall = new IndexAggregateFunction(FunctionNames.MAX_EVER, key, null);
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(1066L);
        recordStore.saveRecord(recBuilder.build());
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        final Tuple expected = indexTypes == MinMaxIndexTypes.TUPLE ? Tuple.fromList(Collections.singletonList(null)) : null;
        assertEquals(expected, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(expected, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        commit(context);
    }
}

Example 8 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxIndex.

@ParameterizedTest(name = "minMaxIndex({0})")
@EnumSource(MinMaxIndexTypes.class)
public void minMaxIndex(MinMaxIndexTypes indexTypes) throws Exception {
    final FieldKeyExpression recno = field("rec_no");
    final GroupingKeyExpression byKey = recno.groupBy(field("num_value_3_indexed"));
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("min", byKey, indexTypes.min()));
        md.addIndex(type, new Index("max", byKey, indexTypes.max()));
    };
    final IndexAggregateFunction minOverall = new IndexAggregateFunction(FunctionNames.MIN_EVER, recno, null);
    final IndexAggregateFunction maxOverall = new IndexAggregateFunction(FunctionNames.MAX_EVER, recno, null);
    final IndexAggregateFunction minByKey = new IndexAggregateFunction(FunctionNames.MIN_EVER, byKey, null);
    final IndexAggregateFunction maxByKey = new IndexAggregateFunction(FunctionNames.MAX_EVER, byKey, null);
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertNull(recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(0, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(99, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(1, recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(96, recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteRecord(Tuple.from(0));
        recordStore.deleteRecord(Tuple.from(99));
        assertEquals(0, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(99, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
    // verify that negatives do not appear in min/max
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(-1);
        recBuilder.setNumValue3Indexed(1);
        recordStore.saveRecord(recBuilder.build());
        if (!indexTypes.shouldAllowNegative()) {
            fail("should have thrown exception");
        }
    } catch (RecordCoreException e) {
        if (indexTypes.shouldAllowNegative()) {
            throw e;
        }
        assertEquals(e.getMessage(), "Attempted update of MAX_EVER_LONG or MIN_EVER_LONG index with negative value");
    }
}