Search in sources :

Example 16 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexerMultiTargetTest method testMultiTargetIndividualContinueAfterCrash.

@Test
public void testMultiTargetIndividualContinueAfterCrash() {
    // After crash, finish building each index individually
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 107;
    final int chunkSize = 17;
    List<Index> indexes = new ArrayList<>();
    // Here: Value indexes only
    indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
    indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
    indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    openSimpleMetaData();
    populateData(numRecords);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
    openSimpleMetaData(hook);
    disableAll(indexes);
    // 1. partly build multi
    buildIndexAndCrashHalfway(chunkSize, 3, timer, OnlineIndexer.newBuilder().setTargetIndexes(indexes));
    // 2. continue each index to done
    for (Index index : indexes) {
        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setIndex(index).setLimit(chunkSize).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).build()) {
            indexBuilder.buildIndex(true);
        }
    }
    // 3. Verify all readable
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        for (Index index : indexes) {
            assertTrue(recordStore.isIndexReadable(index));
        }
        context.commit();
    }
    validateIndexes(indexes);
}
Also used : GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) ArrayList(java.util.ArrayList) Index(com.apple.foundationdb.record.metadata.Index) Test(org.junit.jupiter.api.Test)

Example 17 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexerMultiTargetTest method testMultiTargetContinuation.

@Test
public void testMultiTargetContinuation() {
    // Build the index in small chunks
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 107;
    final int chunkSize = 17;
    final int numChunks = 1 + (numRecords / chunkSize);
    List<Index> indexes = new ArrayList<>();
    indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
    indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
    indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    openSimpleMetaData();
    populateData(numRecords);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
    openSimpleMetaData(hook);
    disableAll(indexes);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setLimit(chunkSize).build()) {
        indexBuilder.buildIndex(true);
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(numChunks, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
    validateIndexes(indexes);
}
Also used : GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) ArrayList(java.util.ArrayList) Index(com.apple.foundationdb.record.metadata.Index) Test(org.junit.jupiter.api.Test)

Example 18 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class OnlineIndexerMultiTargetTest method testMultiTargetPartlyBuildFailure.

@Test
public void testMultiTargetPartlyBuildFailure() {
    // Throw when one index has a different type stamp
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 107;
    final int chunkSize = 17;
    List<Index> indexes = new ArrayList<>();
    indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
    indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
    indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    openSimpleMetaData();
    populateData(numRecords);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
    openSimpleMetaData(hook);
    disableAll(indexes);
    // 1. partly build multi
    buildIndexAndCrashHalfway(chunkSize, 2, timer, OnlineIndexer.newBuilder().setTargetIndexes(indexes));
    // 2. let one index continue ahead
    timer.reset();
    buildIndexAndCrashHalfway(chunkSize, 2, timer, OnlineIndexer.newBuilder().setIndex(indexes.get(2)));
    // 3. assert mismatch type stamp
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setLimit(chunkSize).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).build()) {
        RecordCoreException e = assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
        assertTrue(e.getMessage().contains("This index was partly built by another method"));
    }
}
Also used : RecordCoreException(com.apple.foundationdb.record.RecordCoreException) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) ArrayList(java.util.ArrayList) Index(com.apple.foundationdb.record.metadata.Index) Test(org.junit.jupiter.api.Test)

Example 19 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class LuceneDocumentFromRecordTest method groupingMap.

@Test
void groupingMap() {
    TestRecordsTextProto.MapDocument message = TestRecordsTextProto.MapDocument.newBuilder().setDocId(7).addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("r1").setValue("val").setSecondValue("2val").setThirdValue("3val")).addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("r2").setValue("nval").setSecondValue("2nval").setThirdValue("3nval")).setGroup(30).build();
    FDBRecord<Message> record = unstoredRecord(message);
    KeyExpression index = new GroupingKeyExpression(concat(field("group"), field("entry", KeyExpression.FanType.FanOut).nest(concat(field("key"), function(LuceneFunctionNames.LUCENE_TEXT, field("value")), function(LuceneFunctionNames.LUCENE_TEXT, field("second_value")), function(LuceneFunctionNames.LUCENE_TEXT, field("third_value"))))), 3);
    assertEquals(ImmutableMap.of(Tuple.from(30, "r1"), ImmutableList.of(textField("entry_value", "val"), textField("entry_second_value", "2val"), textField("entry_third_value", "3val")), Tuple.from(30, "r2"), ImmutableList.of(textField("entry_value", "nval"), textField("entry_second_value", "2nval"), textField("entry_third_value", "3nval"))), LuceneDocumentFromRecord.getRecordFields(index, record));
    // Build the partial record message for suggestion
    Descriptors.Descriptor recordDescriptor = message.getDescriptorForType();
    TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder();
    LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "entry_value", "suggestion", Tuple.from(30, "r1"));
    TestRecordsTextProto.MapDocument partialMsg = builder.build();
    assertEquals(1, partialMsg.getEntryCount());
    TestRecordsTextProto.MapDocument.Entry entry = partialMsg.getEntry(0);
    // The suggestion is supposed to show up in value field within repeated entry sub-message
    assertEquals("suggestion", entry.getValue());
    // The second_value field is not supposed to be populated
    assertFalse(entry.hasSecondValue());
    // The key field within repeated entry sub-message is supposed to be populated because it is part of grouping key
    assertEquals("r1", entry.getKey());
    // The group field is supposed to be populated because it is part of grouping key
    assertEquals(30L, partialMsg.getGroup());
}
Also used : TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) Message(com.google.protobuf.Message) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Descriptors(com.google.protobuf.Descriptors) Test(org.junit.jupiter.api.Test)

Example 20 with GroupingKeyExpression

use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method sumBoundIndex.

@Test
public void sumBoundIndex() throws Exception {
    final FieldKeyExpression recno = field("rec_no");
    final GroupingKeyExpression byKey = recno.groupBy(field("num_value_3_indexed"));
    final RecordMetaDataHook hook = md -> md.addUniversalIndex(new Index("sum", byKey, IndexTypes.SUM));
    final IndexAggregateFunction subtotal = new IndexAggregateFunction(FunctionNames.SUM, byKey, null);
    final List<String> allTypes = Collections.emptyList();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        final Optional<IndexAggregateFunction> boundSubtotal = IndexFunctionHelper.bindAggregateFunction(recordStore, subtotal, allTypes, IndexQueryabilityFilter.DEFAULT);
        assertTrue(boundSubtotal.isPresent(), "should find a suitable index");
        assertEquals("sum", boundSubtotal.get().getIndex());
        final Optional<IndexAggregateGroupKeys> keyFunction = IndexAggregateGroupKeys.conditionsToGroupKeys(subtotal, Query.field("num_value_3_indexed").equalsValue(1));
        assertTrue(keyFunction.isPresent(), "should match conditions");
        final Key.Evaluated keys = keyFunction.get().getGroupKeys(recordStore, EvaluationContext.EMPTY);
        assertEquals(Key.Evaluated.scalar(1), keys);
        assertEquals((99 * 100) / (2 * 5) - 20, recordStore.evaluateAggregateFunction(allTypes, subtotal, keys, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals((99 * 100) / (2 * 5) - 20, recordStore.evaluateAggregateFunction(allTypes, boundSubtotal.get(), keys, IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) IndexScanType(com.apple.foundationdb.record.IndexScanType) Pair(org.apache.commons.lang3.tuple.Pair) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Set(java.util.Set) FanType(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType) TupleRange(com.apple.foundationdb.record.TupleRange) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) InvalidIndexEntry(com.apple.foundationdb.record.provider.foundationdb.indexes.InvalidIndexEntry) AutoContinuingCursor(com.apple.foundationdb.record.cursors.AutoContinuingCursor) Matchers.is(org.hamcrest.Matchers.is) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) ArrayList(java.util.ArrayList) Strings(com.google.common.base.Strings) CloseableAsyncIterator(com.apple.foundationdb.async.CloseableAsyncIterator) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) FDBRecordStoreBase.indexEntryKey(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.indexEntryKey) Nullable(javax.annotation.Nullable) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Tags(com.apple.test.Tags) TestRecords1EvolvedProto(com.apple.foundationdb.record.TestRecords1EvolvedProto) ExecutionException(java.util.concurrent.ExecutionException) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) ThenKeyExpression(com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression) IndexEntry(com.apple.foundationdb.record.IndexEntry) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) LoggerFactory(org.slf4j.LoggerFactory) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) Random(java.util.Random) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) ImmutableSet(com.google.common.collect.ImmutableSet) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Collection(java.util.Collection) CompletionException(java.util.concurrent.CompletionException) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) TypeSafeMatcher(org.hamcrest.TypeSafeMatcher) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) TestNoIndexesProto(com.apple.foundationdb.record.TestNoIndexesProto) LazyCursor(com.apple.foundationdb.record.cursors.LazyCursor) EnumSource(org.junit.jupiter.params.provider.EnumSource) CompletableFuture(java.util.concurrent.CompletableFuture) Iterators(com.google.common.collect.Iterators) Key(com.apple.foundationdb.record.metadata.Key) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) HashSet(java.util.HashSet) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) BooleanSource(com.apple.test.BooleanSource) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) Matchers.hasEntry(org.hamcrest.Matchers.hasEntry) Description(org.hamcrest.Description) Matchers.oneOf(org.hamcrest.Matchers.oneOf) Logger(org.slf4j.Logger) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) IndexState(com.apple.foundationdb.record.IndexState) TestRecordsIndexFilteringProto(com.apple.foundationdb.record.TestRecordsIndexFilteringProto) Message(com.google.protobuf.Message) Collections(java.util.Collections) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Index(com.apple.foundationdb.record.metadata.Index) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) FieldKeyExpression(com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression) FDBRecordStoreBase.indexEntryKey(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase.indexEntryKey) Key(com.apple.foundationdb.record.metadata.Key) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Aggregations

GroupingKeyExpression (com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression)57 Index (com.apple.foundationdb.record.metadata.Index)41 Test (org.junit.jupiter.api.Test)39 KeyExpression (com.apple.foundationdb.record.metadata.expressions.KeyExpression)35 ArrayList (java.util.ArrayList)29 Nonnull (javax.annotation.Nonnull)25 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)23 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)22 List (java.util.List)22 RecordMetaData (com.apple.foundationdb.record.RecordMetaData)21 Message (com.google.protobuf.Message)21 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)21 RecordMetaDataBuilder (com.apple.foundationdb.record.RecordMetaDataBuilder)20 IndexAggregateFunction (com.apple.foundationdb.record.metadata.IndexAggregateFunction)20 Expressions.field (com.apple.foundationdb.record.metadata.Key.Expressions.field)20 EmptyKeyExpression (com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression)20 Query (com.apple.foundationdb.record.query.expressions.Query)20 Tuple (com.apple.foundationdb.tuple.Tuple)20 Tags (com.apple.test.Tags)20 Tag (org.junit.jupiter.api.Tag)20