Examples with GroupingKeyExpression com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression used on opensource projects

Example 11 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method enableRecordVersionsAfterTheFact.

@ParameterizedTest(name = "enableRecordVersionsAfterTheFact [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void enableRecordVersionsAfterTheFact(int testFormatVersion, boolean testSplitLongRecords) throws ExecutionException, InterruptedException {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(871L).setNumValue2(871).build();
    MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1415).build();
    MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(3415L).setNumValue2(3415).build();
    Index globalCountIndex = new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
    RecordMetaDataHook origHook = metaDataBuilder -> {
        noVersionHook.apply(metaDataBuilder);
        metaDataBuilder.addIndex((RecordTypeBuilder) null, globalCountIndex);
    };
    try (FDBRecordContext context = openContext(origHook)) {
        assertFalse(metaData.isStoreRecordVersions());
        recordStore.saveRecord(record1);
        recordStore.saveRecord(record2);
        recordStore.saveRecord(record3, null, FDBRecordStoreBase.VersionstampSaveBehavior.WITH_VERSION);
        context.commit();
    }
    try (FDBRecordContext context = openContext(metaDataBuilder -> {
        origHook.apply(metaDataBuilder);
        metaDataBuilder.setStoreRecordVersions(true);
        functionVersionHook.apply(metaDataBuilder);
    })) {
        assertTrue(metaData.isStoreRecordVersions());
        FDBStoredRecord<Message> storedRecord1 = recordStore.loadRecord(Tuple.from(871L));
        assertNotNull(storedRecord1);
        assertEquals(record1, storedRecord1.getRecord());
        assertFalse(storedRecord1.hasVersion());
        FDBStoredRecord<Message> storedRecord2 = recordStore.loadRecord(Tuple.from(1415L));
        assertNotNull(storedRecord2);
        assertEquals(record2, storedRecord2.getRecord());
        assertFalse(storedRecord2.hasVersion());
        FDBStoredRecord<Message> storedRecord3 = recordStore.loadRecord(Tuple.from(3415L));
        assertNotNull(storedRecord3);
        assertEquals(record3, storedRecord3.getRecord());
        assertTrue(storedRecord3.hasVersion());
        RecordCursor<IndexEntry> cursor = recordStore.scanIndex(metaData.getIndex("MySimpleRecord$maybeVersion"), IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN);
        assertEquals(Arrays.asList(Tuple.from(null, 1415L), Tuple.from(FDBRecordVersion.MIN_VERSION.toVersionstamp(), 871L), Tuple.from(storedRecord3.getVersion().toVersionstamp(), 3415L)), cursor.map(IndexEntry::getKey).asList().get());
    }
}

Example 12 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method withMetaDataRebuilds.

@ParameterizedTest(name = "withMetaDataRebuilds [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void withMetaDataRebuilds(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    RecordMetaDataHook firstHook = metaDataBuilder -> {
        metaDataBuilder.setSplitLongRecords(splitLongRecords);
        metaDataBuilder.addUniversalIndex(new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
        metaDataBuilder.addUniversalIndex(new Index("globalVersion", VersionKeyExpression.VERSION, IndexTypes.VERSION));
    };
    RecordMetaDataHook secondHook = metaDataBuilder -> {
        firstHook.apply(metaDataBuilder);
        metaDataBuilder.removeIndex("globalVersion");
        metaDataBuilder.setStoreRecordVersions(false);
    };
    RecordMetaDataHook thirdHook = metaDataBuilder -> {
        secondHook.apply(metaDataBuilder);
        metaDataBuilder.setStoreRecordVersions(true);
        metaDataBuilder.addUniversalIndex(new Index("globalVersion2", VersionKeyExpression.VERSION, IndexTypes.VERSION));
    };
    MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).build();
    FDBRecordVersion version1;
    MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1776L).build();
    MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(1955L).build();
    FDBRecordVersion version3;
    RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
    // First with version on.
    try (FDBRecordContext context = openContext(firstHook)) {
        FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record1);
        assertTrue(storedRecord.hasVersion());
        context.commit();
        version1 = FDBRecordVersion.complete(context.getVersionStamp(), storedRecord.getVersion().getLocalVersion());
    }
    try (FDBRecordContext context = openContext(firstHook)) {
        FDBStoredRecord<?> loadedRecord = recordStore.loadRecord(Tuple.from(1066L));
        assertNotNull(loadedRecord);
        assertTrue(loadedRecord.hasVersion());
        assertEquals(version1, loadedRecord.getVersion());
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, indexScan("globalVersion"));
        List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
        assertEquals(1, records.size());
        FDBQueriedRecord<Message> queriedRecord = records.get(0);
        assertEquals(Tuple.from(1066L), queriedRecord.getPrimaryKey());
        assertTrue(queriedRecord.hasVersion());
        assertEquals(version1, queriedRecord.getVersion());
    }
    // Now with version off.
    try (FDBRecordContext context = openContext(secondHook)) {
        FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record2);
        assertFalse(storedRecord.hasVersion());
        context.commit();
    }
    try (FDBRecordContext context = openContext(secondHook)) {
        FDBStoredRecord<?> loadedRecord1 = recordStore.loadRecord(Tuple.from(1066L));
        assertNotNull(loadedRecord1);
        assertEquals(testFormatVersion >= FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION, loadedRecord1.hasVersion());
        FDBStoredRecord<?> loadedRecord2 = recordStore.loadRecord(Tuple.from(1776L));
        assertNotNull(loadedRecord2);
        assertFalse(loadedRecord2.hasVersion());
        assertThrows(RecordCoreException.class, () -> {
            RecordQueryPlan plan = planner.plan(query);
            fail("Came up with plan " + plan.toString() + " when it should be impossible");
        });
    }
    // Now with version back on.
    try (FDBRecordContext context = openContext(thirdHook)) {
        FDBStoredRecord<?> storedRecord = recordStore.saveRecord(record3);
        assertTrue(storedRecord.hasVersion());
        context.commit();
        version3 = FDBRecordVersion.complete(context.getVersionStamp(), storedRecord.getVersion().getLocalVersion());
    }
    try (FDBRecordContext context = openContext(thirdHook)) {
        FDBStoredRecord<?> loadedRecord1 = recordStore.loadRecord(Tuple.from(1066L));
        assertEquals(testFormatVersion >= FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION, loadedRecord1.hasVersion());
        FDBStoredRecord<?> loadedRecord2 = recordStore.loadRecord(Tuple.from(1776L));
        assertFalse(loadedRecord2.hasVersion());
        FDBStoredRecord<?> loadedRecord3 = recordStore.loadRecord(Tuple.from(1955L));
        assertTrue(loadedRecord3.hasVersion());
        assertEquals(version3, loadedRecord3.getVersion());
        RecordQueryPlan plan = planner.plan(query);
        assertThat(plan, indexScan("globalVersion2"));
        List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
        assertEquals(3, records.size());
        if (testFormatVersion < FDBRecordStore.SAVE_VERSION_WITH_RECORD_FORMAT_VERSION) {
            FDBQueriedRecord<Message> queriedRecord1 = records.get(0);
            assertEquals(Tuple.from(1066L), queriedRecord1.getPrimaryKey());
            assertFalse(queriedRecord1.hasVersion());
            FDBQueriedRecord<Message> queriedRecord2 = records.get(1);
            assertEquals(Tuple.from(1776L), queriedRecord2.getPrimaryKey());
            assertFalse(queriedRecord2.hasVersion());
        } else {
            FDBQueriedRecord<Message> queriedRecord1 = records.get(0);
            assertEquals(Tuple.from(1776L), queriedRecord1.getPrimaryKey());
            assertFalse(queriedRecord1.hasVersion());
            FDBQueriedRecord<Message> queriedRecord2 = records.get(1);
            assertEquals(Tuple.from(1066L), queriedRecord2.getPrimaryKey());
            assertTrue(queriedRecord2.hasVersion());
            assertEquals(version1, queriedRecord2.getVersion());
        }
        FDBQueriedRecord<Message> queriedRecord3 = records.get(2);
        assertEquals(Tuple.from(1955L), queriedRecord3.getPrimaryKey());
        assertTrue(queriedRecord3.hasVersion());
        assertEquals(version3, queriedRecord3.getVersion());
    }
}

Example 13 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxTupleRepeatConcatenated.

@Test
public void minMaxTupleRepeatConcatenated() throws Exception {
    final FieldKeyExpression fieldKey = field("repeater", FanType.Concatenate);
    final GroupingKeyExpression indexKey = fieldKey.ungrouped();
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("min", indexKey, IndexTypes.MIN_EVER_TUPLE));
        md.addIndex(type, new Index("max", indexKey, IndexTypes.MAX_EVER_TUPLE));
    };
    final IndexAggregateFunction min = new IndexAggregateFunction(FunctionNames.MIN_EVER, indexKey, null);
    final IndexAggregateFunction max = new IndexAggregateFunction(FunctionNames.MAX_EVER, indexKey, null);
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertNull(recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
        recBuilder.setRecNo(1);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        recBuilder.addRepeater(1);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from(1L)), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        recBuilder.addRepeater(1);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from(1L, 1L)), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        recBuilder.clearRepeater();
        recBuilder.addRepeater(2);
        recordStore.saveRecord(recBuilder.build());
        assertEquals(Tuple.from(Tuple.from()), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertEquals(Tuple.from(Tuple.from(2L)), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        commit(context);
    }
}

Example 14 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method sizeBasedUserVersionChecker.

@Test
public void sizeBasedUserVersionChecker() {
    final Index universalCountIndex = new Index("countIndex", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
    final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
    metaDataBuilder.addUniversalIndex(universalCountIndex);
    final RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
    // Open the store and make sure estimate size path is used
    try (FDBRecordContext context = openContext()) {
        final SizeBasedUserVersionChecker userVersionChecker = new SizeBasedUserVersionChecker(IndexState.READABLE);
        final FDBRecordStore recordStore = storeBuilder(context, metaData1).setUserVersionChecker(userVersionChecker).create();
        assertThat("should have checked the size at least once", userVersionChecker.getCheckSizeCount(), greaterThan(0L));
        assertEquals(1, timer.getCount(FDBStoreTimer.Events.ESTIMATE_SIZE));
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable(), "all indexes should be readable on new store opening");
        commit(context);
    }
    // Re-open the store and make sure the estimate is never used
    try (FDBRecordContext context = openContext()) {
        final SizeBasedUserVersionChecker userVersionChecker = new SizeBasedUserVersionChecker(IndexState.DISABLED);
        final FDBRecordStore recordStore = storeBuilder(context, metaData1).setUserVersionChecker(userVersionChecker).open();
        assertEquals(0, userVersionChecker.getCheckSizeCount(), "should not have checked the size on already created store");
        assertEquals(0, timer.getCount(FDBStoreTimer.Events.ESTIMATE_SIZE), "should not have estimated the size on already created store");
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable(), "all indexes should be readable on already created store opening");
        commit(context);
    }
    // Add two indexes
    final String indexName1 = "index1";
    final String indexName2 = "index2";
    metaDataBuilder.addIndex("MySimpleRecord", indexName1, "num_value_2");
    metaDataBuilder.addIndex("MySimpleRecord", indexName2, "num_value_2");
    final RecordMetaData metaData2 = metaDataBuilder.getRecordMetaData();
    try (FDBRecordContext context = openContext()) {
        final SizeBasedUserVersionChecker userVersionChecker = new SizeBasedUserVersionChecker(IndexState.DISABLED);
        final FDBRecordStore recordStore = storeBuilder(context, metaData2).setUserVersionChecker(userVersionChecker).open();
        assertEquals(2, userVersionChecker.getCheckSizeCount(), "should have checked the size once for each index");
        assertEquals(1, timer.getCount(FDBStoreTimer.Events.ESTIMATE_SIZE), "should have only checked the database for the size once");
        assertEquals(ImmutableSet.of(indexName1, indexName2), recordStore.getRecordStoreState().getDisabledIndexNames());
        commit(context);
    }
}

Example 15 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexerMultiTargetTest method testMultiTargetRebuild.

@Test
public void testMultiTargetRebuild() {
    // Use inline rebuildIndex
    final FDBStoreTimer timer = new FDBStoreTimer();
    final long numRecords = 80;
    List<Index> indexes = new ArrayList<>();
    indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
    indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
    openSimpleMetaData();
    populateData(numRecords);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
    openSimpleMetaData(hook);
    disableAll(indexes);
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).build()) {
            indexBuilder.rebuildIndex(recordStore);
        }
        for (Index index : indexes) {
            recordStore.markIndexReadable(index).join();
        }
        context.commit();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    validateIndexes(indexes);
}