Examples with GroupingKeyExpression com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression used on opensource projects

Example 36 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexerMultiTargetTest method testMultiTargetSimple.

@Test
public void testMultiTargetSimple() {
    // Simply build the index
    final FDBStoreTimer timer = new FDBStoreTimer();
    final long numRecords = 80;
    List<Index> indexes = new ArrayList<>();
    // Here: Value indexes only
    indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
    indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    openSimpleMetaData();
    populateData(numRecords);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
    openSimpleMetaData(hook);
    disableAll(indexes);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).build()) {
        indexBuilder.buildIndex(true);
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        for (Index index : indexes) {
            assertTrue(recordStore.isIndexReadable(index));
        }
        context.commit();
    }
    // Here: Add a non-value index
    indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
    // Test building non-idempotent + force
    timer.reset();
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfReadable(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD).build()).build()) {
        indexBuilder.buildIndex(true);
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        for (Index index : indexes) {
            assertTrue(recordStore.isIndexReadable(index));
        }
        context.commit();
    }
    // Now use an arbitrary primary index
    timer.reset();
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfReadable(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD).build()).build()) {
        indexBuilder.buildIndex(true);
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        for (Index index : indexes) {
            assertTrue(recordStore.isIndexReadable(index));
        }
        context.commit();
    }
    validateIndexes(indexes);
}

Example 37 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexerMultiTargetTest method testMultiTargetPartlyBuildChangeTargets.

@Test
void testMultiTargetPartlyBuildChangeTargets() {
    // Throw when the index list changes
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 107;
    final int chunkSize = 17;
    List<Index> indexes = new ArrayList<>();
    indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
    indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
    indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    openSimpleMetaData();
    populateData(numRecords);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
    openSimpleMetaData(hook);
    disableAll(indexes);
    // 1. partly build multi
    buildIndexAndCrashHalfway(chunkSize, 2, timer, OnlineIndexer.newBuilder().setTargetIndexes(indexes));
    // 2. Change indexes set
    indexes.remove(2);
    // 3. assert mismatch type stamp
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setLimit(chunkSize).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).build()) {
        RecordCoreException e = assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
        assertTrue(e.getMessage().contains("This index was partly built by another method"));
    }
}

Example 38 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexScrubberTest method testScrubberInvalidIndexState.

@Test
void testScrubberInvalidIndexState() {
    final int numRecords = 20;
    Index tgtIndex = new Index("tgt_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(tgtIndex);
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildIndex(tgtIndex);
    // refuse to scrub a non-readable index
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).build()) {
            recordStore.markIndexWriteOnly(tgtIndex).join();
            context.commit();
            assertThrows(IndexingBase.ValidationException.class, indexScrubber::scrubDanglingIndexEntries);
            assertThrows(IndexingBase.ValidationException.class, indexScrubber::scrubMissingIndexEntries);
        }
    }
    // refuse to scrub a non-value index
    Index nonValueIndex = new Index("count_index", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
    FDBRecordStoreTestBase.RecordMetaDataHook hook2 = myHook(nonValueIndex);
    openSimpleMetaData(hook2);
    buildIndex(nonValueIndex);
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(nonValueIndex).setSubspace(subspace).build()) {
        assertThrows(IndexingBase.ValidationException.class, indexScrubber::scrubDanglingIndexEntries);
        assertThrows(IndexingBase.ValidationException.class, indexScrubber::scrubMissingIndexEntries);
    }
}

Example 39 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class FunctionKeyIndexTest method testCountIndex.

@Test
public void testCountIndex() throws Exception {
    Index index = new Index("group_index", new GroupingKeyExpression(function("substr", concat(field("str_value"), value(0), value(3))), 0), IndexTypes.COUNT);
    Records records = Records.create();
    for (int i = 0; i < 10; i++) {
        records.add(i, "ab" + Character.toString((char) ('c' + (i % 3))) + "_" + i);
    }
    saveRecords(records, index);
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, index);
        List<IndexEntry> keyValues = getIndexKeyValues(recordStore, index, IndexScanType.BY_GROUP);
        assertEquals(Tuple.from("abc"), keyValues.get(0).getKey());
        assertEquals(Tuple.from(4), keyValues.get(0).getValue());
        assertEquals(Tuple.from("abd"), keyValues.get(1).getKey());
        assertEquals(Tuple.from(3), keyValues.get(1).getValue());
        assertEquals(Tuple.from("abe"), keyValues.get(2).getKey());
        assertEquals(Tuple.from(3), keyValues.get(2).getValue());
    }
}

Example 40 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class IndexFunctionHelperTest method groupSubKeysNested.

@Test
void groupSubKeysNested() {
    final KeyExpression wholeKey = concat(field("a", FanOut).nest(concatenateFields("b", "c")), field("d"), field("e", FanOut).nest(concatenateFields("f", "g")));
    final KeyExpression group0 = new GroupingKeyExpression(wholeKey, 0);
    assertEquals(wholeKey, IndexFunctionHelper.getGroupingKey(group0));
    assertEquals(empty(), IndexFunctionHelper.getGroupedKey(group0));
    final KeyExpression group1 = new GroupingKeyExpression(wholeKey, 1);
    assertEquals(concat(field("a", FanOut).nest(concatenateFields("b", "c")), field("d"), field("e", FanOut).nest(field("f"))), IndexFunctionHelper.getGroupingKey(group1));
    assertEquals(field("e", FanOut).nest(field("g")), IndexFunctionHelper.getGroupedKey(group1));
    final KeyExpression group2 = new GroupingKeyExpression(wholeKey, 2);
    assertEquals(concat(field("a", FanOut).nest(concatenateFields("b", "c")), field("d")), IndexFunctionHelper.getGroupingKey(group2));
    assertEquals(field("e", FanOut).nest(concatenateFields("f", "g")), IndexFunctionHelper.getGroupedKey(group2));
    final KeyExpression group3 = new GroupingKeyExpression(wholeKey, 3);
    assertEquals(field("a", FanOut).nest(concatenateFields("b", "c")), IndexFunctionHelper.getGroupingKey(group3));
    assertEquals(concat(field("d"), field("e", FanOut).nest(concatenateFields("f", "g"))), IndexFunctionHelper.getGroupedKey(group3));
    final KeyExpression group4 = new GroupingKeyExpression(wholeKey, 4);
    assertEquals(field("a", FanOut).nest(field("b")), IndexFunctionHelper.getGroupingKey(group4));
    assertEquals(concat(field("a", FanOut).nest(field("c")), field("d"), field("e", FanOut).nest(concatenateFields("f", "g"))), IndexFunctionHelper.getGroupedKey(group4));
}