Examples with IndexTypes com.apple.foundationdb.record.metadata.IndexTypes used on opensource projects

Example 1 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method testOnlineIndexerBuilderWriteLimitBytes.

@Test
public void testOnlineIndexerBuilderWriteLimitBytes() throws Exception {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("newIndex", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    final FDBStoreTimer timer = new FDBStoreTimer();
    try (FDBRecordContext context = openContext()) {
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        // Build in this transaction.
        try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setRecordStore(recordStore).setTimer(timer).setIndex("newIndex").setLimit(100000).setMaxWriteLimitBytes(1).build()) {
            // this call will "flatten" the staccato iterations to a whole range. Testing compatibility.
            indexer.rebuildIndex(recordStore);
        }
        recordStore.markIndexReadable("newIndex").join();
        assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
        assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
        assertEquals(199, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
        // last item
        assertEquals(1, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        assertTrue(recordStore.isIndexReadable("newIndex"));
        recordStore.clearAndMarkIndexWriteOnly("newIndex").join();
        context.commit();
    }
    timer.reset();
    try (FDBRecordContext context = openContext()) {
        recordStore.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS).join();
        // Build in this transaction.
        try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setRecordStore(recordStore).setTimer(timer).setIndex("newIndex").setLimit(100000).setMaxWriteLimitBytes(1).build()) {
            Key.Evaluated key = indexer.buildUnbuiltRange(Key.Evaluated.scalar(0L), Key.Evaluated.scalar(25L)).join();
            assertEquals(1, key.getLong(0));
            assertEquals(1, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
            assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
        }
        recordStore.clearAndMarkIndexWriteOnly("newIndex").join();
        context.commit();
    }
    timer.reset();
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setTimer(timer).setIndex(index).setLimit(100000).setMaxWriteLimitBytes(1).setRecordsPerSecond(OnlineIndexer.UNLIMITED).build()) {
        indexer.buildIndex();
    }
    assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(200, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    // this includes two endpoints + one range = total of 3 terminations by count
    // - note that (last, null] endpoint is en empty range
    assertEquals(3, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
    // this is the range between the endpoints - 199 items in (first, last] interval
    assertEquals(198, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_SIZE));
}

Example 2 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method testMarkReadableClearsBuiltRanges.

@Test
public void testMarkReadableClearsBuiltRanges() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("newIndex", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        indexer.buildIndex(true);
    }
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        // Verify rangeSet is cleared when index is marked readable
        final RangeSet rangeSet = new RangeSet(recordStore.indexRangeSubspace(index));
        AsyncIterator<Range> ranges = rangeSet.missingRanges(recordStore.ensureContextActive()).iterator();
        final Range range = ranges.next();
        final boolean range1IsEmpty = RangeSet.isFirstKey(range.begin) && RangeSet.isFinalKey(range.end);
        assertTrue(range1IsEmpty);
        // fake commit, happy compiler
        context.commit();
    }
}

Example 3 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildRangeTransactional.

@Test
public void buildRangeTransactional() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit changes
        }
        return ret;
    };
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        try (FDBRecordContext context = openContext()) {
            recordStore.markIndexWriteOnly(index).join();
            context.commit();
        }
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, null, null).join();
            indexBuilder.buildRange(recordStore, null, null).join();
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
        clearIndexData(index);
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, null, Key.Evaluated.scalar(130L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(8515L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), Key.Evaluated.scalar(130L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(8515L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), Key.Evaluated.scalar(150L)).join();
            context.commit();
        }
        assertEquals(Tuple.from(11325L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(100L), null).join();
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
        clearIndexData(index);
        try (FDBRecordContext context = openContext()) {
            for (long l = 0L; l < 200L; l += 10) {
                indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(l), Key.Evaluated.scalar(l + 5L)).join();
            }
            context.commit();
        }
        assertEquals(Tuple.from(9800L), getAggregate.get());
        try (FDBRecordContext context = openContext()) {
            for (long l = 0L; l < 200L; l += 10) {
                indexBuilder.buildRange(recordStore, null, null).join();
            }
            context.commit();
        }
        assertEquals(Tuple.from(20100L), getAggregate.get());
    }
}

Example 4 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method buildRangeWithNull.

@Test
public void buildRangeWithNull() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("simple$value_2", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    final Supplier<Tuple> getAggregate = () -> {
        Tuple ret;
        try (FDBRecordContext context = openContext()) {
            assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).join());
            FDBRecordStore recordStore2 = recordStore.asBuilder().setContext(context).uncheckedOpen();
            ret = recordStore2.evaluateAggregateFunction(indexTypes, aggregateFunction, TupleRange.ALL, IsolationLevel.SERIALIZABLE).join();
        // Do NOT commit changes
        }
        return ret;
    };
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        recordStore.markIndexWriteOnly(index).join();
        context.commit();
    }
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).build()) {
        indexBuilder.buildRange(null, null).join();
        assertEquals(Tuple.from(20100L), getAggregate.get());
    }
}

Example 5 with IndexTypes

use of com.apple.foundationdb.record.metadata.IndexTypes in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method testTimeLimit.

@Test
public void testTimeLimit() {
    List<TestRecords1Proto.MySimpleRecord> records = LongStream.range(0, 200).mapToObj(val -> TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(val).setNumValue2((int) val + 1).build()).collect(Collectors.toList());
    Index index = new Index("newIndex", field("num_value_2").ungrouped(), IndexTypes.SUM);
    IndexAggregateFunction aggregateFunction = new IndexAggregateFunction(FunctionNames.SUM, index.getRootExpression(), index.getName());
    List<String> indexTypes = Collections.singletonList("MySimpleRecord");
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> metaDataBuilder.addIndex("MySimpleRecord", index);
    openSimpleMetaData();
    try (FDBRecordContext context = openContext()) {
        records.forEach(recordStore::saveRecord);
        context.commit();
    }
    openSimpleMetaData(hook);
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setTimeLimitMilliseconds(1).setLimit(20).setConfigLoader(old -> {
        // Ensure that time limit is exceeded
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
            fail("The test was interrupted");
        }
        return old;
    }).build()) {
        IndexingBase.TimeLimitException e = assertThrows(IndexingBase.TimeLimitException.class, indexer::buildIndex);
        assertTrue(e.getMessage().contains("Time Limit Exceeded"));
    }
}