Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 66 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method uniquenessViolations.

@Test
public void uniquenessViolations() throws Exception {
    final String indexName = "MySimpleRecord$num_value_unique";
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        assertThat(recordStore.isIndexWriteOnly(indexName), is(false));
        recordStore.markIndexWriteOnly(indexName).get();
        assertThat(recordStore.isIndexWriteOnly(indexName), is(true));
        context.commit();
    }
    TestRecords1Proto.MySimpleRecord record1 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValueUnique(42).build();
    TestRecords1Proto.MySimpleRecord record2 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1793L).setNumValueUnique(42).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.saveRecord(record1);
        recordStore.saveRecord(record2);
        context.commit();
    }
    Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$num_value_unique");
    // Test scan uniqueness violations.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        RecordCursorIterator<RecordIndexUniquenessViolation> cursor = recordStore.scanUniquenessViolations(index).asIterator();
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation first = cursor.next();
        assertEquals(Tuple.from(42L), first.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), first.getPrimaryKey());
        assertEquals(Tuple.from(1793L), first.getExistingKey());
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation second = cursor.next();
        assertEquals(Tuple.from(42L), second.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), second.getPrimaryKey());
        assertEquals(Tuple.from(1066L), second.getExistingKey());
        assertFalse(cursor.hasNext());
    }
    // Test scan uniqueness violations given value key.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        RecordCursorIterator<RecordIndexUniquenessViolation> cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(42)).asIterator();
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation first = cursor.next();
        assertEquals(Tuple.from(42L), first.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), first.getPrimaryKey());
        assertEquals(Tuple.from(1793L), first.getExistingKey());
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation second = cursor.next();
        assertEquals(Tuple.from(42L), second.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), second.getPrimaryKey());
        assertEquals(Tuple.from(1066L), second.getExistingKey());
        assertFalse(cursor.hasNext());
    }
    // Test requesting to resolve uniqueness violations with a remaining record.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.resolveUniquenessViolation(index, Tuple.from(42), Tuple.from(1066L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNotNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
    }
    // Test requesting to resolve uniqueness violations with no remaining record.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.resolveUniquenessViolation(index, Tuple.from(42), null).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
    }
    // Test manually resolving uniqueness violations by deleting one record.
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        recordStore.deleteRecordAsync(Tuple.from(1793L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNotNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
    }
}

Example 67 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method orphanedIndexEntry.

@Test
public void orphanedIndexEntry() throws Exception {
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1).setStrValueIndexed("foo").setNumValueUnique(1).build());
        recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(2).setStrValueIndexed("bar").setNumValueUnique(2).build());
        recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(3).setStrValueIndexed("baz").setNumValueUnique(3).build());
        commit(context);
    }
    // Delete the "bar" record with the index removed.
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context, builder -> {
            builder.removeIndex("MySimpleRecord$str_value_indexed");
        });
        recordStore.deleteRecord(Tuple.from(2));
        commit(context);
    }
    // associated record.
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        // Verify our entries
        Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
        assertTrue(recordStore.hasIndexEntryRecord(new IndexEntry(index, Tuple.from("foo", 1), TupleHelpers.EMPTY), IsolationLevel.SERIALIZABLE).get(), "'Foo' should exist");
        assertFalse(recordStore.hasIndexEntryRecord(new IndexEntry(index, Tuple.from("bar", 2), TupleHelpers.EMPTY), IsolationLevel.SERIALIZABLE).get(), "'Bar' should be deleted");
        assertTrue(recordStore.hasIndexEntryRecord(new IndexEntry(index, Tuple.from("baz", 3), TupleHelpers.EMPTY), IsolationLevel.SERIALIZABLE).get(), "'Baz' should exist");
        try {
            recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed").asList().get();
            fail("Scan should have found orphaned record");
        } catch (ExecutionException e) {
            assertEquals("record not found from index entry", e.getCause().getMessage());
        }
        commit(context);
    }
    // Try again, but this time scan allowing orphaned entries.
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        List<FDBIndexedRecord<Message>> records = recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, TupleRange.ALL, null, IndexOrphanBehavior.RETURN, ScanProperties.FORWARD_SCAN).asList().get();
        assertEquals(records.size(), 3);
        for (FDBIndexedRecord<Message> record : records) {
            if (record.getIndexEntry().getKey().getString(0).equals("bar")) {
                assertFalse(record.hasStoredRecord(), "Entry for 'bar' should be orphaned");
                assertThrows(RecordCoreException.class, record::getStoredRecord);
                assertThrows(RecordCoreException.class, record::getRecord);
            } else {
                assertTrue(record.hasStoredRecord(), "Entry for '" + record.getIndexEntry().getKey() + "' should have an associated record");
            }
        }
        commit(context);
    }
    // Try once again, but this time skipping orphaned entries
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        List<FDBIndexedRecord<Message>> records = recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, TupleRange.ALL, null, IndexOrphanBehavior.SKIP, ScanProperties.FORWARD_SCAN).asList().get();
        assertEquals(records.size(), 2);
        for (FDBIndexedRecord<Message> record : records) {
            assertNotEquals("bar", record.getIndexEntry().getKey().getString(0));
            assertTrue(record.hasStoredRecord(), "Entry for '" + record.getIndexEntry().getKey() + "' should have an associated record");
        }
        commit(context);
    }
    // Validate the index. Should only return the index entry that has no associated record.
    try (FDBRecordContext context = openContext()) {
        uncheckedOpenSimpleRecordStore(context);
        final Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
        final List<InvalidIndexEntry> invalidIndexEntries = recordStore.getIndexMaintainer(index).validateEntries(null, null).asList().get();
        assertEquals(Collections.singletonList(InvalidIndexEntry.newOrphan(new IndexEntry(index, Tuple.from("bar", 2), TupleHelpers.EMPTY))), invalidIndexEntries, "Validation should return the index entry that has no associated record.");
        commit(context);
    }
}

Example 68 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method countValueIndex.

@Test
public void countValueIndex() throws Exception {
    final FieldKeyExpression numValue3 = field("num_value_3_indexed");
    final GroupingKeyExpression byKey = numValue3.groupBy(field("str_value_indexed"));
    final RecordMetaDataHook hook = md -> md.addIndex("MySimpleRecord", new Index("count_num_3", byKey, IndexTypes.COUNT_NOT_NULL));
    final List<String> types = Collections.singletonList("MySimpleRecord");
    final IndexAggregateFunction perKey = new IndexAggregateFunction(FunctionNames.COUNT_NOT_NULL, byKey, null);
    final IndexAggregateFunction total = new IndexAggregateFunction(FunctionNames.COUNT_NOT_NULL, numValue3, null);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            if (i % 5 == 0) {
                recBuilder.clearNumValue3Indexed();
            } else {
                recBuilder.setNumValue3Indexed(i + 1000);
            }
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(80, recordStore.evaluateAggregateFunction(types, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(40, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteRecord(Tuple.from(8));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(79, recordStore.evaluateAggregateFunction(types, total, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(39, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}

Example 69 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method countClearWhenZero.

@ParameterizedTest
@BooleanSource
public void countClearWhenZero(boolean clearWhenZero) throws Exception {
    final GroupingKeyExpression byKey = new GroupingKeyExpression(field("str_value_indexed"), 0);
    final RecordMetaDataHook hook = md -> md.addIndex("MySimpleRecord", new Index("count_by_str", byKey, IndexTypes.COUNT, ImmutableMap.of(IndexOptions.CLEAR_WHEN_ZERO, Boolean.toString(clearWhenZero))));
    final List<String> types = Collections.singletonList("MySimpleRecord");
    final IndexAggregateFunction perKey = new IndexAggregateFunction(FunctionNames.COUNT, byKey, null);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 10; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(5, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(5, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("odd"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(ImmutableMap.of("even", 5L, "odd", 5L), recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("count_by_str"), IndexScanType.BY_GROUP, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(i -> Pair.of(i.getKey().get(0), i.getValue().get(0))).asList().join().stream().collect(Collectors.toMap(Pair::getLeft, Pair::getRight)));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 10; i += 2) {
            recordStore.deleteRecord(Tuple.from(i));
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(0, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("even"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(5, recordStore.evaluateAggregateFunction(types, perKey, Key.Evaluated.scalar("odd"), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(clearWhenZero ? ImmutableMap.of("odd", 5L) : ImmutableMap.of("even", 0L, "odd", 5L), recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("count_by_str"), IndexScanType.BY_GROUP, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).map(i -> Pair.of(i.getKey().get(0), i.getValue().get(0))).asList().join().stream().collect(Collectors.toMap(Pair::getLeft, Pair::getRight)));
        commit(context);
    }
}

Example 70 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testIndexMissingValidation.

@Test
public void testIndexMissingValidation() throws Exception {
    final int nRecords = 10;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int i = 0; i < nRecords; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed(Integer.toString(i));
            // nRecords is not larger than 10, so the indexes (sorted by the string version of recNo) are in the
            // same order as the records. This can make the test easy.
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    // Delete the indexes of some records.
    Set<InvalidIndexEntry> expectedInvalidEntries = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        final Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
        openSimpleRecordStore(context);
        List<FDBStoredRecord<Message>> savedRecords = recordStore.scanRecords(TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        List<IndexEntry> indexEntries = recordStore.scanIndex(index, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        for (int i = 0; i < nRecords; i += 2) {
            IndexEntry indexEntry = indexEntries.get(i);
            FDBStoredRecord<Message> record = savedRecords.get(i);
            final Tuple valueKey = indexEntry.getKey();
            final Tuple entryKey = indexEntryKey(index, valueKey, record.getPrimaryKey());
            final byte[] keyBytes = recordStore.indexSubspace(index).pack(valueKey);
            byte[] v0 = recordStore.getContext().ensureActive().get(keyBytes).get();
            recordStore.getContext().ensureActive().clear(keyBytes);
            byte[] v = recordStore.getContext().ensureActive().get(keyBytes).get();
            expectedInvalidEntries.add(InvalidIndexEntry.newMissing(indexEntry, record));
        }
        commit(context);
    }
    try (FDBDatabaseRunner runner = fdb.newRunner()) {
        AtomicInteger generatorCount = new AtomicInteger();
        // Set a scanned records limit to mock when the NoNextReason is out of band.
        RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).openAsync().thenApply(currentRecordStore -> {
            generatorCount.getAndIncrement();
            final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
            ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
            return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
        }))).asIterator();
        Set<InvalidIndexEntry> results = new HashSet<>();
        cursor.forEachRemaining(results::add);
        assertEquals(expectedInvalidEntries, results);
        // The number of scans is about the number of index entries (orphan validation) plus the number of records
        // (missing validation).
        assertThat(generatorCount.get(), greaterThanOrEqualTo((5 + 10) / 4));
    }
}