use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method minMaxTupleRepeated.
@Test
public void minMaxTupleRepeated() throws Exception {
final FieldKeyExpression fieldKey = field("repeater", FanType.FanOut);
final GroupingKeyExpression indexKey = fieldKey.ungrouped();
final RecordMetaDataHook hook = md -> {
RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
md.addIndex(type, new Index("min", indexKey, IndexTypes.MIN_EVER_TUPLE));
md.addIndex(type, new Index("max", indexKey, IndexTypes.MAX_EVER_TUPLE));
};
final IndexAggregateFunction min = new IndexAggregateFunction(FunctionNames.MIN_EVER, indexKey, null);
final IndexAggregateFunction max = new IndexAggregateFunction(FunctionNames.MAX_EVER, indexKey, null);
List<String> types = Collections.singletonList("MySimpleRecord");
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
assertNull(recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(1);
recordStore.saveRecord(recBuilder.build());
assertNull(recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
recBuilder.addRepeater(1);
recordStore.saveRecord(recBuilder.build());
assertEquals(Tuple.from(1L), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertEquals(Tuple.from(1L), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
recBuilder.clearRepeater();
recBuilder.addRepeater(2);
recBuilder.addRepeater(3);
recordStore.saveRecord(recBuilder.build());
assertEquals(Tuple.from(1L), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertEquals(Tuple.from(3L), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
recBuilder.clearRepeater();
recBuilder.addRepeater(-1);
recordStore.saveRecord(recBuilder.build());
assertEquals(Tuple.from(-1L), recordStore.evaluateAggregateFunction(types, min, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertEquals(Tuple.from(3L), recordStore.evaluateAggregateFunction(types, max, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
commit(context);
}
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method minMaxIndex.
@ParameterizedTest(name = "minMaxIndex({0})")
@EnumSource(MinMaxIndexTypes.class)
public void minMaxIndex(MinMaxIndexTypes indexTypes) throws Exception {
final FieldKeyExpression recno = field("rec_no");
final GroupingKeyExpression byKey = recno.groupBy(field("num_value_3_indexed"));
final RecordMetaDataHook hook = md -> {
RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
md.addIndex(type, new Index("min", byKey, indexTypes.min()));
md.addIndex(type, new Index("max", byKey, indexTypes.max()));
};
final IndexAggregateFunction minOverall = new IndexAggregateFunction(FunctionNames.MIN_EVER, recno, null);
final IndexAggregateFunction maxOverall = new IndexAggregateFunction(FunctionNames.MAX_EVER, recno, null);
final IndexAggregateFunction minByKey = new IndexAggregateFunction(FunctionNames.MIN_EVER, byKey, null);
final IndexAggregateFunction maxByKey = new IndexAggregateFunction(FunctionNames.MAX_EVER, byKey, null);
List<String> types = Collections.singletonList("MySimpleRecord");
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
assertNull(recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
assertNull(recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
for (int i = 0; i < 100; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(i);
recBuilder.setNumValue3Indexed(i % 5);
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
assertEquals(0, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(99, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(1, recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(96, recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
commit(context);
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.deleteRecord(Tuple.from(0));
recordStore.deleteRecord(Tuple.from(99));
assertEquals(0, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
assertEquals(99, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
commit(context);
}
// verify that negatives do not appear in min/max
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(-1);
recBuilder.setNumValue3Indexed(1);
recordStore.saveRecord(recBuilder.build());
if (!indexTypes.shouldAllowNegative()) {
fail("should have thrown exception");
}
} catch (RecordCoreException e) {
if (indexTypes.shouldAllowNegative()) {
throw e;
}
assertEquals(e.getMessage(), "Attempted update of MAX_EVER_LONG or MIN_EVER_LONG index with negative value");
}
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class OnlineIndexerMultiTargetTest method testMultiTargetMismatchStateFailure.
@Test
public void testMultiTargetMismatchStateFailure() {
// Throw when one index has a different status
final FDBStoreTimer timer = new FDBStoreTimer();
final long numRecords = 3;
List<Index> indexes = new ArrayList<>();
// Here: Value indexes only
indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
openSimpleMetaData();
populateData(numRecords);
FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
openSimpleMetaData(hook);
disableAll(indexes);
// built one index
try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(indexes.get(1)).setSubspace(subspace).build()) {
indexer.buildIndex(false);
}
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).build()) {
RecordCoreException e = assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
assertTrue(e.getMessage().contains("A target index state doesn't match the primary index state"));
}
assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
// Now finish with REBUILD
timer.reset();
openSimpleMetaData(hook);
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfWriteOnly(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD).build()).build()) {
indexBuilder.buildIndex(true);
}
assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
validateIndexes(indexes);
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class OnlineIndexerMultiTargetTest method benchMarkMultiTarget.
// uncomment to compare
// @Test
public void benchMarkMultiTarget() {
// compare single target build to multi target index building
final FDBStoreTimer singleTimer = new FDBStoreTimer();
final FDBStoreTimer multiTimer = new FDBStoreTimer();
final int numRecords = 5555;
List<Index> indexes = new ArrayList<>();
indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
openSimpleMetaData();
populateData(numRecords);
FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
long startSingle = System.currentTimeMillis();
for (Index index : indexes) {
openSimpleMetaData(hook);
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTimer(singleTimer).setIndex(index).build()) {
indexBuilder.buildIndex(true);
}
}
long endSingle = System.currentTimeMillis();
// suppress warnings
assertTrue(endSingle > startSingle);
disableAll(indexes);
long startMulti = System.currentTimeMillis();
openSimpleMetaData(hook);
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(multiTimer).build()) {
indexBuilder.buildIndex(true);
}
long endMulti = System.currentTimeMillis();
// suppress warnings
assertTrue(endMulti > startMulti);
System.out.printf("%d indexes, %d records. Single build took %d milliSeconds, MultiIndex took %d%n", indexes.size(), numRecords, endSingle - startSingle, endMulti - startMulti);
}
use of com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression in project fdb-record-layer by FoundationDB.
the class OnlineIndexerMultiTargetTest method testMultiTargetContinueAfterCrash.
@Test
void testMultiTargetContinueAfterCrash() {
// Crash, then continue successfully
final FDBStoreTimer timer = new FDBStoreTimer();
final int numRecords = 107;
final int chunkSize = 17;
List<Index> indexes = new ArrayList<>();
indexes.add(new Index("indexB", field("num_value_3_indexed"), IndexTypes.VALUE));
indexes.add(new Index("indexA", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
indexes.add(new Index("indexD", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT));
indexes.add(new Index("indexC", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
openSimpleMetaData();
populateData(numRecords);
FDBRecordStoreTestBase.RecordMetaDataHook hook = allIndexesHook(indexes);
openSimpleMetaData(hook);
disableAll(indexes);
// 1. partly build multi
buildIndexAndCrashHalfway(chunkSize, 5, timer, OnlineIndexer.newBuilder().setTargetIndexes(indexes));
// 2. continue and done
try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTargetIndexes(indexes).setTimer(timer).setLimit(chunkSize).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).build()) {
indexBuilder.buildIndex(true);
}
assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
openSimpleMetaData(hook);
try (FDBRecordContext context = openContext()) {
for (Index index : indexes) {
assertTrue(recordStore.isIndexReadable(index));
}
context.commit();
}
validateIndexes(indexes);
}
Aggregations