use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class OnlineIndexerMultiTargetTest method testMultiTargetIndividualContinueAfterCrash.
@Test
public void testMultiTargetIndividualContinueAfterCrash() {
// After crash, finish building each index individually
final FDBStoreTimer timer = new FDBStoreTimer();
final int numRecords = 107;
final int chunkSize = 17;
List<Index> indexes = new ArrayList<>();
// Here: Value indexes only
indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
openSimpleMetaData();
populateData(numRecords);
FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
openSimpleMetaData(hook);
disableAll(indexes);
// 1. partly build multi
buildIndexAndCrashHalfway(chunkSize, 3, timer, OnlineIndexer.newBuilder().setTargetIndexes(indexes));
// 2. continue each index to done
for (Index index : indexes) {
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setIndex(index).setLimit(chunkSize).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).build()) {
indexBuilder.buildIndex(true);
}
}
// 3. Verify all readable
openSimpleMetaData(hook);
try (FDBRecordContext context = openContext()) {
for (Index index : indexes) {
assertTrue(recordStore.isIndexReadable(index));
}
context.commit();
}
validateIndexes(indexes);
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class OnlineIndexerMultiTargetTest method testMultiTargetContinuation.
@Test
public void testMultiTargetContinuation() {
// Build the index in small chunks
final FDBStoreTimer timer = new FDBStoreTimer();
final int numRecords = 107;
final int chunkSize = 17;
final int numChunks = 1 + (numRecords / chunkSize);
List<Index> indexes = new ArrayList<>();
indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
openSimpleMetaData();
populateData(numRecords);
FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
openSimpleMetaData(hook);
disableAll(indexes);
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setLimit(chunkSize).build()) {
indexBuilder.buildIndex(true);
}
assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
assertEquals(numChunks, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
validateIndexes(indexes);
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class OnlineIndexerMultiTargetTest method testMultiTargetPartlyBuildFailure.
@Test
public void testMultiTargetPartlyBuildFailure() {
// Throw when one index has a different type stamp
final FDBStoreTimer timer = new FDBStoreTimer();
final int numRecords = 107;
final int chunkSize = 17;
List<Index> indexes = new ArrayList<>();
indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
openSimpleMetaData();
populateData(numRecords);
FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
openSimpleMetaData(hook);
disableAll(indexes);
// 1. partly build multi
buildIndexAndCrashHalfway(chunkSize, 2, timer, OnlineIndexer.newBuilder().setTargetIndexes(indexes));
// 2. let one index continue ahead
timer.reset();
buildIndexAndCrashHalfway(chunkSize, 2, timer, OnlineIndexer.newBuilder().setIndex(indexes.get(2)));
// 3. assert mismatch type stamp
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setLimit(chunkSize).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).build()) {
RecordCoreException e = assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
assertTrue(e.getMessage().contains("This index was partly built by another method"));
}
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class LuceneDocumentFromRecordTest method groupingMap.
@Test
void groupingMap() {
TestRecordsTextProto.MapDocument message = TestRecordsTextProto.MapDocument.newBuilder().setDocId(7).addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("r1").setValue("val").setSecondValue("2val").setThirdValue("3val")).addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("r2").setValue("nval").setSecondValue("2nval").setThirdValue("3nval")).setGroup(30).build();
FDBRecord<Message> record = unstoredRecord(message);
KeyExpression index = new GroupingKeyExpression(concat(field("group"), field("entry", KeyExpression.FanType.FanOut).nest(concat(field("key"), function(LuceneFunctionNames.LUCENE_TEXT, field("value")), function(LuceneFunctionNames.LUCENE_TEXT, field("second_value")), function(LuceneFunctionNames.LUCENE_TEXT, field("third_value"))))), 3);
assertEquals(ImmutableMap.of(Tuple.from(30, "r1"), ImmutableList.of(textField("entry_value", "val"), textField("entry_second_value", "2val"), textField("entry_third_value", "3val")), Tuple.from(30, "r2"), ImmutableList.of(textField("entry_value", "nval"), textField("entry_second_value", "2nval"), textField("entry_third_value", "3nval"))), LuceneDocumentFromRecord.getRecordFields(index, record));
// Build the partial record message for suggestion
Descriptors.Descriptor recordDescriptor = message.getDescriptorForType();
TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder();
LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "entry_value", "suggestion", Tuple.from(30, "r1"));
TestRecordsTextProto.MapDocument partialMsg = builder.build();
assertEquals(1, partialMsg.getEntryCount());
TestRecordsTextProto.MapDocument.Entry entry = partialMsg.getEntry(0);
// The suggestion is supposed to show up in value field within repeated entry sub-message
assertEquals("suggestion", entry.getValue());
// The second_value field is not supposed to be populated
assertFalse(entry.hasSecondValue());
// The key field within repeated entry sub-message is supposed to be populated because it is part of grouping key
assertEquals("r1", entry.getKey());
// The group field is supposed to be populated because it is part of grouping key
assertEquals(30L, partialMsg.getGroup());
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method sumBoundIndex.
@Test
public void sumBoundIndex() throws Exception {
final FieldKeyExpression recno = field("rec_no");
final GroupingKeyExpression byKey = recno.groupBy(field("num_value_3_indexed"));
final RecordMetaDataHook hook = md -> md.addUniversalIndex(new Index("sum", byKey, IndexTypes.SUM));
final IndexAggregateFunction subtotal = new IndexAggregateFunction(FunctionNames.SUM, byKey, null);
final List<String> allTypes = Collections.emptyList();
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
for (int i = 0; i < 100; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(i);
recBuilder.setNumValue3Indexed(i % 5);
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
final Optional<IndexAggregateFunction> boundSubtotal = IndexFunctionHelper.bindAggregateFunction(recordStore, subtotal, allTypes, IndexQueryabilityFilter.DEFAULT);
assertTrue(boundSubtotal.isPresent(), "should find a suitable index");
assertEquals("sum", boundSubtotal.get().getIndex());
final Optional<IndexAggregateGroupKeys> keyFunction = IndexAggregateGroupKeys.conditionsToGroupKeys(subtotal, Query.field("num_value_3_indexed").equalsValue(1));
assertTrue(keyFunction.isPresent(), "should match conditions");
final Key.Evaluated keys = keyFunction.get().getGroupKeys(recordStore, EvaluationContext.EMPTY);
assertEquals(Key.Evaluated.scalar(1), keys);
assertEquals((99 * 100) / (2 * 5) - 20, recordStore.evaluateAggregateFunction(allTypes, subtotal, keys, IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals((99 * 100) / (2 * 5) - 20, recordStore.evaluateAggregateFunction(allTypes, boundSubtotal.get(), keys, IsolationLevel.SNAPSHOT).join().getLong(0));
commit(context);
}
}
Aggregations