Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 61 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method sumIndex.

@Test
public void sumIndex() throws Exception {
    final FieldKeyExpression recno = field("rec_no");
    final GroupingKeyExpression byKey = recno.groupBy(field("num_value_3_indexed"));
    final RecordMetaDataHook hook = md -> md.addUniversalIndex(new Index("sum", byKey, IndexTypes.SUM));
    final IndexAggregateFunction subtotal = new IndexAggregateFunction(FunctionNames.SUM, byKey, null);
    final IndexAggregateFunction total = new IndexAggregateFunction(FunctionNames.SUM, recno, null);
    final List<String> allTypes = Collections.emptyList();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(0L, recordStore.evaluateAggregateFunction(allTypes, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(0L, recordStore.evaluateAggregateFunction(allTypes, subtotal, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals((99 * 100) / 2, recordStore.evaluateAggregateFunction(allTypes, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals((99 * 100) / (2 * 5) - 20, recordStore.evaluateAggregateFunction(allTypes, subtotal, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteRecord(Tuple.from(10));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals((99 * 100) / 2 - 10, recordStore.evaluateAggregateFunction(allTypes, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}

Example 62 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method markReadable.

@Test
public void markReadable() throws Exception {
    final String indexName = "MySimpleRecord$str_value_indexed";
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        assertThat(recordStore.isIndexWriteOnly(indexName), is(false));
        recordStore.clearAndMarkIndexWriteOnly(indexName).get();
        assertThat(recordStore.isIndexWriteOnly(indexName), is(true));
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        Index index = recordStore.getRecordMetaData().getIndex(indexName);
        assertThat(recordStore.isIndexReadable(index), is(false));
        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore).setIndex(index).build()) {
            indexBuilder.buildRange(recordStore, null, Key.Evaluated.scalar(1066L)).get();
            Optional<Range> firstUnbuilt = recordStore.firstUnbuiltRange(index).get();
            assertTrue(firstUnbuilt.isPresent());
            assertArrayEquals(Key.Evaluated.scalar(1066).toTuple().pack(), firstUnbuilt.get().begin);
            assertArrayEquals(new byte[] { (byte) 0xff }, firstUnbuilt.get().end);
            recordStore.markIndexReadable(index).handle((built, e) -> {
                assertNotNull(e);
                assertThat(e, instanceOf(CompletionException.class));
                assertNotNull(e.getCause());
                assertThat(e.getCause(), instanceOf(FDBRecordStore.IndexNotBuiltException.class));
                return null;
            }).get();
            assertThat(recordStore.isIndexReadable(index), is(false));
            indexBuilder.buildRange(recordStore, Key.Evaluated.scalar(1066L), null).get();
            assertFalse(recordStore.firstUnbuiltRange(index).get().isPresent());
            assertTrue(recordStore.markIndexReadable(index).get());
            assertFalse(recordStore.markIndexReadable(index).get());
            assertThat(recordStore.isIndexReadable(index), is(true));
        }
    // Not committing.
    }
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        Index index = recordStore.getRecordMetaData().getIndex(indexName);
        assertThat(recordStore.isIndexReadable(index), is(false));
        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setRecordStore(recordStore).setIndex(index).build()) {
            indexBuilder.buildRange(recordStore, null, Key.Evaluated.scalar(1066L)).get();
        }
        Optional<Range> firstUnbuilt = recordStore.firstUnbuiltRange(index).get();
        assertTrue(firstUnbuilt.isPresent());
        assertArrayEquals(Key.Evaluated.scalar(1066).toTuple().pack(), firstUnbuilt.get().begin);
        assertArrayEquals(new byte[] { (byte) 0xff }, firstUnbuilt.get().end);
        assertTrue(recordStore.uncheckedMarkIndexReadable(index.getName()).get());
        assertFalse(recordStore.uncheckedMarkIndexReadable(index.getName()).get());
        assertThat(recordStore.isIndexReadable(index), is(true));
        // Purposefully, checking to mark an index readable that is already
        // readable does not throw an error.
        firstUnbuilt = recordStore.firstUnbuiltRange(index).get();
        assertTrue(firstUnbuilt.isPresent());
        assertArrayEquals(Key.Evaluated.scalar(1066).toTuple().pack(), firstUnbuilt.get().begin);
        assertArrayEquals(new byte[] { (byte) 0xff }, firstUnbuilt.get().end);
        assertFalse(recordStore.markIndexReadable(index.getName()).get());
        assertThat(recordStore.isIndexReadable(index), is(true));
    }
}

Example 63 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method countMultiValueIndex.

@Test
public void countMultiValueIndex() throws Exception {
    final ThenKeyExpression values = concatenateFields("num_value_2", "num_value_3_indexed");
    final GroupingKeyExpression byKey = values.groupBy(field("str_value_indexed"));
    final RecordMetaDataHook hook = md -> md.addIndex("MySimpleRecord", new Index("count_num_3", byKey, IndexTypes.COUNT_NOT_NULL));
    final List<String> types = Collections.singletonList("MySimpleRecord");
    final IndexAggregateFunction perKey = new IndexAggregateFunction(FunctionNames.COUNT_NOT_NULL, byKey, null);
    final IndexAggregateFunction total = new IndexAggregateFunction(FunctionNames.COUNT_NOT_NULL, values, null);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            if (i % 3 == 0) {
                recBuilder.clearNumValue2();
            } else {
                recBuilder.setNumValue2(i + 1000);
            }
            if (i % 5 == 0) {
                recBuilder.clearNumValue3Indexed();
            } else {
                recBuilder.setNumValue3Indexed(i + 10000);
            }
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(100 - 100 / 5 - 100 / 3 + 100 / 15, recordStore.evaluateAggregateFunction(types, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(50 - 50 / 5 - 50 / 3 + 50 / 15, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}

Example 64 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method noMaintenanceFilteredOnIndex.

@Test
public void noMaintenanceFilteredOnIndex() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openAnyRecordStore(TestRecordsIndexFilteringProto.getDescriptor(), context);
        TestRecordsIndexFilteringProto.MyBasicRecord recordA = TestRecordsIndexFilteringProto.MyBasicRecord.newBuilder().setRecNo(1001).setNumValue2(101).build();
        recordStore.saveRecord(recordA);
        context.commit();
        Collection<StoreTimer.Event> events = context.getTimer().getEvents();
        assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
        assertFalse(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
    }
    // add index
    RecordMetaDataHook hook = metaDataBuilder -> {
        metaDataBuilder.setVersion(metaDataBuilder.getVersion() + 1);
        metaDataBuilder.addIndex("MyBasicRecord", "value2$filtered", field("num_value_2"));
    };
    // rebuild the index so that it is in a READABLE state
    try (FDBRecordContext context = openContext()) {
        openAnyRecordStore(TestRecordsIndexFilteringProto.getDescriptor(), context, hook);
        try (OnlineIndexer indexer = OnlineIndexer.forRecordStoreAndIndex(recordStore, "value2$filtered")) {
            indexer.buildIndex();
        }
    }
    try (FDBRecordContext context = openContext()) {
        openAnyRecordStore(TestRecordsIndexFilteringProto.getDescriptor(), context, hook);
        context.getTimer().reset();
        TestRecordsIndexFilteringProto.MyBasicRecord recordA = TestRecordsIndexFilteringProto.MyBasicRecord.newBuilder().setRecNo(1002).setNumValue2(102).build();
        recordStore.saveRecord(recordA);
        context.commit();
        Collection<StoreTimer.Event> events = context.getTimer().getEvents();
        assertTrue(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
        assertFalse(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
    }
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder metaData = RecordMetaData.newBuilder().setRecords(TestRecordsIndexFilteringProto.getDescriptor());
        hook.apply(metaData);
        IndexMaintenanceFilter noneFilter = (i, r) -> IndexMaintenanceFilter.IndexValues.NONE;
        recordStore = getStoreBuilder(context, metaData.getRecordMetaData()).setIndexMaintenanceFilter(noneFilter).createOrOpen();
        setupPlanner(null);
        context.getTimer().reset();
        TestRecordsIndexFilteringProto.MyBasicRecord recordB = TestRecordsIndexFilteringProto.MyBasicRecord.newBuilder().setRecNo(1003).setNumValue2(103).build();
        recordStore.saveRecord(recordB);
        context.commit();
        Collection<StoreTimer.Event> events = context.getTimer().getEvents();
        assertFalse(events.contains(FDBStoreTimer.Events.SAVE_INDEX_ENTRY));
        assertTrue(events.contains(FDBStoreTimer.Events.SKIP_INDEX_RECORD));
    }
}

Example 65 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testIndexOrphanValidationByAutoContinuingCursor.

@Test
public void testIndexOrphanValidationByAutoContinuingCursor() throws Exception {
    Set<IndexEntry> expectedInvalidEntries = setUpIndexOrphanValidation();
    try (FDBDatabaseRunner runner = fdb.newRunner()) {
        AtomicInteger generatorCount = new AtomicInteger();
        // Set a scanned records limit to mock when the transaction is out of band.
        RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).uncheckedOpenAsync().thenApply(currentRecordStore -> {
            generatorCount.getAndIncrement();
            final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
            ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
            return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
        }))).asIterator();
        Set<IndexEntry> results = new HashSet<>();
        while (cursor.hasNext()) {
            InvalidIndexEntry invalidIndexEntry = cursor.next();
            assertEquals(InvalidIndexEntry.Reasons.ORPHAN, invalidIndexEntry.getReason());
            IndexEntry entry = invalidIndexEntry.getEntry();
            assertFalse(results.contains(entry), "Entry " + entry + " is duplicated");
            results.add(entry);
        }
        assertEquals(expectedInvalidEntries, results, "Validation should return the index entry that has no associated record.");
        // The number of scans is about the number of index entries (orphan validation) plus the number of records
        // (missing validation).
        assertThat(generatorCount.get(), greaterThanOrEqualTo((20 + 10) / 4));
    }
}