Search in sources :

Example 11 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method testMarkReadableClearsBuiltRanges.

@Test
public void testMarkReadableClearsBuiltRanges() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("newIndex", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        indexer.buildIndex(true);
    }
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        // Verify rangeSet is cleared when index is marked readable
        final RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(index));
        AsyncIterator<Range> ranges = rangeSet.missingRanges(recordStore.ensureContextActive()).iterator();
        final Range range = ranges.next();
        final boolean range1IsEmpty = RangeSet.isFirstKey(range.begin) && RangeSet.isFinalKey(range.end);
        assertTrue(range1IsEmpty);
        // fake commit, happy compiler
        context.commit();
    }
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ThreadContext(org.apache.logging.log4j.ThreadContext) DO_NOT_RE_INCREASE_LIMIT(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DO_NOT_RE_INCREASE_LIMIT) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Collectors(java.util.stream.Collectors) DEFAULT_PROGRESS_LOG_INTERVAL(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DEFAULT_PROGRESS_LOG_INTERVAL) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Queue(java.util.Queue) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) Function(java.util.function.Function) Supplier(java.util.function.Supplier) Key(com.apple.foundationdb.record.metadata.Key) RecordCoreRetriableTransactionException(com.apple.foundationdb.record.RecordCoreRetriableTransactionException) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Matchers.oneOf(org.hamcrest.Matchers.oneOf) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) LongStream(java.util.stream.LongStream) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) ExecutionException(java.util.concurrent.ExecutionException) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) Collections(java.util.Collections) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test)

Example 12 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildRangeTransactional.

@Test
public void buildRangeTransactional() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit changes
        }
        return ret;
    };
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        try (FDBRecordContext context = openContext()) {
            recordStore.markIndexWriteOnly(index).join();
            context.commit();
        }
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, null, null).join();
            indexBuilder.buildRange(recordStore, null, null).join();
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
        clearIndexData(index);
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, null, Key.Evaluated.scalar(130L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(8515L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), Key.Evaluated.scalar(130L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(8515L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), Key.Evaluated.scalar(150L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(11325L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), null).join();
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
        clearIndexData(index);
        try (FDBRecordContext context = openContext()) {
            for (long l = 0L; l < 200L; l += 10) {
                indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(l), Key.Evaluated.scalar(l + 5L)).join();
            }
            context.commit();
        }
        assertEquals(Tuple.from(9800L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            for (long l = 0L; l < 200L; l += 10) {
                indexBuilder.buildRange(recordStore, null, null).join();
            }
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
    }
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ThreadContext(org.apache.logging.log4j.ThreadContext) DO_NOT_RE_INCREASE_LIMIT(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DO_NOT_RE_INCREASE_LIMIT) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Collectors(java.util.stream.Collectors) DEFAULT_PROGRESS_LOG_INTERVAL(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DEFAULT_PROGRESS_LOG_INTERVAL) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Queue(java.util.Queue) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) Function(java.util.function.Function) Supplier(java.util.function.Supplier) Key(com.apple.foundationdb.record.metadata.Key) RecordCoreRetriableTransactionException(com.apple.foundationdb.record.RecordCoreRetriableTransactionException) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Matchers.oneOf(org.hamcrest.Matchers.oneOf) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) LongStream(java.util.stream.LongStream) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) ExecutionException(java.util.concurrent.ExecutionException) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) Collections(java.util.Collections) Index(com.apple.foundationdb.record.metadata.Index) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Tuple(com.apple.foundationdb.tuple.Tuple) Test(org.junit.jupiter.api.Test)

Example 13 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildRangeWithNull.

@Test
public void buildRangeWithNull() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit changes
        }
        return ret;
    };
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        recordStore.markIndexWriteOnly(index).join();
        context.commit();
    }
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        indexBuilder.buildRange(null, null).join();
        assertEquals(Tuple.from(20100L), getAggregate.get());
    }
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ThreadContext(org.apache.logging.log4j.ThreadContext) DO_NOT_RE_INCREASE_LIMIT(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DO_NOT_RE_INCREASE_LIMIT) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Collectors(java.util.stream.Collectors) DEFAULT_PROGRESS_LOG_INTERVAL(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DEFAULT_PROGRESS_LOG_INTERVAL) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Queue(java.util.Queue) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) Function(java.util.function.Function) Supplier(java.util.function.Supplier) Key(com.apple.foundationdb.record.metadata.Key) RecordCoreRetriableTransactionException(com.apple.foundationdb.record.RecordCoreRetriableTransactionException) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Matchers.oneOf(org.hamcrest.Matchers.oneOf) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) LongStream(java.util.stream.LongStream) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) ExecutionException(java.util.concurrent.ExecutionException) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) Collections(java.util.Collections) Index(com.apple.foundationdb.record.metadata.Index) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Tuple(com.apple.foundationdb.tuple.Tuple) Test(org.junit.jupiter.api.Test)

Example 14 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method testTimeLimit.

@Test
public void testTimeLimit() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("newIndex", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setTimeLimitMilliseconds(1).setLimit(20).setConfigLoader(old -> {
        // Ensure that time limit is exceeded
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
            fail("The test was interrupted");
        }
        return old;
    }).build()) {
        IndexingBase.TimeLimitException e = assertThrows(IndexingBase.TimeLimitException.class, indexer::buildIndex);
        assertTrue(e.getMessage().contains("Time Limit Exceeded"));
    }
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ThreadContext(org.apache.logging.log4j.ThreadContext) DO_NOT_RE_INCREASE_LIMIT(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DO_NOT_RE_INCREASE_LIMIT) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Collectors(java.util.stream.Collectors) DEFAULT_PROGRESS_LOG_INTERVAL(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DEFAULT_PROGRESS_LOG_INTERVAL) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Queue(java.util.Queue) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) Function(java.util.function.Function) Supplier(java.util.function.Supplier) Key(com.apple.foundationdb.record.metadata.Key) RecordCoreRetriableTransactionException(com.apple.foundationdb.record.RecordCoreRetriableTransactionException) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Matchers.oneOf(org.hamcrest.Matchers.oneOf) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) LongStream(java.util.stream.LongStream) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) ExecutionException(java.util.concurrent.ExecutionException) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) Collections(java.util.Collections) Index(com.apple.foundationdb.record.metadata.Index) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Test(org.junit.jupiter.api.Test)

Example 15 with IndexAggregateFunction

use of com.apple.foundationdb.record.metadata.IndexAggregateFunction in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildEndpointIdempotency.

@Test
public void buildEndpointIdempotency() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 10).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit the change.
        }
        return ret;
    };
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        recordStore.markIndexWriteOnly(index).join();
        context.commit();
    }
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        final RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(index));
        // Build the endpoints
        TupleRange range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(10L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(0L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(9L).pack(), null).join());
        List<Range> middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(0L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(9L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Make sure running this again doesn't change anything.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(10L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(0L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(9L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(0L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(9L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Remove the first and last records.
        try (FDBRecordContext context = openContext()) {
            recordStore.deleteRecord(Tuple.from(0L));
            recordStore.deleteRecord(Tuple.from(9L));
            context.commit();
        }
        assertEquals(Tuple.from(0L), getAggregate.get());
        // Rerun endpoints with new data.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(1L), range.getLow());
        assertEquals(Tuple.from(8L), range.getHigh());
        assertEquals(Tuple.from(9L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Run it again to show that nothing has happened.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(1L), range.getLow());
        assertEquals(Tuple.from(8L), range.getHigh());
        assertEquals(Tuple.from(9L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Add back the previous first and last records.
        try (FDBRecordContext context = openContext()) {
            recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(0L).setNumValue2(1).build());
            recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(9L).setNumValue2(10).build());
            context.commit();
        }
        assertEquals(Tuple.from(20L), getAggregate.get());
        // Rerun endpoints with new data.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(20L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Run it again to show that nothing has happened.
        range = indexBuilder.buildEndpoints().join();
        assertEquals(Tuple.from(0L), range.getLow());
        assertEquals(Tuple.from(9L), range.getHigh());
        assertEquals(Tuple.from(20L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), null, Tuple.from(1L).pack()).join());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database(), Tuple.from(8L).pack(), null).join());
        middleRanges = rangeSet.missingRanges(fdb.database()).join();
        assertEquals(Collections.singletonList(Tuple.from(1L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.begin)).collect(Collectors.toList()));
        assertEquals(Collections.singletonList(Tuple.from(8L)), middleRanges.stream().map(r -> Tuple.fromBytes(r.end)).collect(Collectors.toList()));
        // Straight up build the whole index.
        indexBuilder.buildIndex(false);
        assertEquals(Tuple.from(55L), getAggregate.get());
        assertEquals(Collections.emptyList(), rangeSet.missingRanges(fdb.database()).join());
    }
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) Pair(org.apache.commons.lang3.tuple.Pair) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ThreadContext(org.apache.logging.log4j.ThreadContext) DO_NOT_RE_INCREASE_LIMIT(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DO_NOT_RE_INCREASE_LIMIT) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Collectors(java.util.stream.Collectors) DEFAULT_PROGRESS_LOG_INTERVAL(com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer.DEFAULT_PROGRESS_LOG_INTERVAL) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Queue(java.util.Queue) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) FunctionNames(com.apple.foundationdb.record.FunctionNames) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RangeSet(com.apple.foundationdb.async.RangeSet) Function(java.util.function.Function) Supplier(java.util.function.Supplier) Key(com.apple.foundationdb.record.metadata.Key) RecordCoreRetriableTransactionException(com.apple.foundationdb.record.RecordCoreRetriableTransactionException) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Matchers.oneOf(org.hamcrest.Matchers.oneOf) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) LongStream(java.util.stream.LongStream) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) ExecutionException(java.util.concurrent.ExecutionException) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) Collections(java.util.Collections) RangeSet(com.apple.foundationdb.async.RangeSet) Index(com.apple.foundationdb.record.metadata.Index) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) Range(com.apple.foundationdb.Range) TupleRange(com.apple.foundationdb.record.TupleRange) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Test(org.junit.jupiter.api.Test)

Aggregations

IndexAggregateFunction (com.apple.foundationdb.record.metadata.IndexAggregateFunction)39 Test (org.junit.jupiter.api.Test)35 Index (com.apple.foundationdb.record.metadata.Index)33 List (java.util.List)30 TupleRange (com.apple.foundationdb.record.TupleRange)29 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)29 FunctionNames (com.apple.foundationdb.record.FunctionNames)28 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)28 Expressions.field (com.apple.foundationdb.record.metadata.Key.Expressions.field)28 Tuple (com.apple.foundationdb.tuple.Tuple)28 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)28 Collections (java.util.Collections)27 Nonnull (javax.annotation.Nonnull)27 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)25 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)25 Key (com.apple.foundationdb.record.metadata.Key)25 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)25 Assertions.assertThrows (org.junit.jupiter.api.Assertions.assertThrows)25 TestRecords1Proto (com.apple.foundationdb.record.TestRecords1Proto)24 CompletableFuture (java.util.concurrent.CompletableFuture)24