Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 71 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method uniquenessViolationsWithFanOut.

@Test
public void uniquenessViolationsWithFanOut() throws Exception {
    Index index = new Index("MySimpleRecord$repeater", field("repeater", FanType.FanOut), Index.EMPTY_VALUE, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", index);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertThat(recordStore.isIndexWriteOnly(index), is(false));
        recordStore.markIndexWriteOnly(index).get();
        assertThat(recordStore.isIndexWriteOnly(index), is(true));
        context.commit();
    }
    TestRecords1Proto.MySimpleRecord record1 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).addRepeater(1).addRepeater(2).addRepeater(3).build();
    TestRecords1Proto.MySimpleRecord record2 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1793L).addRepeater(2).addRepeater(3).build();
    TestRecords1Proto.MySimpleRecord record3 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1849L).addRepeater(3).build();
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.saveRecord(record1);
        recordStore.saveRecord(record2);
        recordStore.saveRecord(record3);
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(5, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(1)).getCount().get());
        assertEquals(2, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(2)).getCount().get());
        assertEquals(3, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(3)).getCount().get());
        RecordCursorIterator<RecordIndexUniquenessViolation> cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(3)).asIterator();
        assertTrue(cursor.hasNext());
        RecordIndexUniquenessViolation next = cursor.next();
        assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), next.getPrimaryKey());
        assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1793L), Tuple.from(1849L))));
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), next.getPrimaryKey());
        assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1066L), Tuple.from(1849L))));
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1849L), next.getPrimaryKey());
        assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1066L), Tuple.from(1793L))));
        assertFalse(cursor.hasNext());
        cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(2)).asIterator();
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(2L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1066L), next.getPrimaryKey());
        assertEquals(Tuple.from(1793L), next.getExistingKey());
        assertTrue(cursor.hasNext());
        next = cursor.next();
        assertEquals(Tuple.from(2L), next.getIndexEntry().getKey());
        assertEquals(Tuple.from(1793L), next.getPrimaryKey());
        assertEquals(Tuple.from(1066L), next.getExistingKey());
        assertFalse(cursor.hasNext());
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Key.Evaluated.scalar(3), null).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1066L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNotNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1793L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNotNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1849L)).get();
        assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
        assertNull(recordStore.loadRecord(Tuple.from(1066L)));
        assertNull(recordStore.loadRecord(Tuple.from(1793L)));
        assertNotNull(recordStore.loadRecord(Tuple.from(1849L)));
    }
}

Example 72 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method assertMaxVersionsForGroups.

@SuppressWarnings("try")
private void assertMaxVersionsForGroups(@Nonnull SortedMap<Integer, FDBRecordVersion> groupsToVersions) {
    try (FDBRecordContext context = openContext(maxEverVersionWithGroupingHook)) {
        Index index = metaData.getIndex("max_ever_version_with_grouping");
        List<IndexEntry> entries = new ArrayList<>(groupsToVersions.size());
        for (Map.Entry<Integer, FDBRecordVersion> mapEntry : groupsToVersions.entrySet()) {
            entries.add(new IndexEntry(index, Key.Evaluated.scalar(mapEntry.getKey()), Key.Evaluated.scalar(mapEntry.getValue())));
        }
        assertMaxVersionEntries(index, entries);
    }
}

Example 73 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method queryOnVersion.

@ParameterizedTest(name = "queryOnVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnVersion(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
    List<TestRecords1Proto.MyOtherRecord> otherRecords = IntStream.range(0, 30).mapToObj(id -> TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(id * 2 + 1).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
    Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
    Iterator<TestRecords1Proto.MyOtherRecord> otherIterator = otherRecords.iterator();
    while (simpleIterator.hasNext()) {
        try (FDBRecordContext context = openContext(simpleVersionHook)) {
            int done = 0;
            while (simpleIterator.hasNext() && done != 5) {
                recordStore.saveRecord(simpleIterator.next());
                recordStore.saveRecord(otherIterator.next());
                done += 1;
            }
            context.commit();
        }
    }
    // Query all records.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = Stream.concat(simpleRecords.stream().map(MySimpleRecord::getRecNo), otherRecords.stream().map(TestRecords1Proto.MyOtherRecord::getRecNo)).sorted().collect(Collectors.toList());
        FDBRecordVersion last = null;
        List<Long> receivedKeys = new ArrayList<>();
        int totalSeen = 0;
        while (true) {
            RecordQueryPlan plan;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
                plan = planner.plan(query);
                assertEquals("Index(globalVersion <,>)", plan.toString());
            } else {
                RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().greaterThan(last)).setSort(VersionKeyExpression.VERSION).build();
                plan = planner.plan(query);
                assertEquals("Index(globalVersion ([" + last.toVersionstamp() + "],>)", plan.toString());
            }
            RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator();
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBQueriedRecord<Message> record = cursor.next();
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals(simpleRecords.size() + otherRecords.size(), totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query MySimpleRecord based on value.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
        List<Long> receivedKeys = new ArrayList<>();
        FDBRecordVersion last = null;
        int totalSeen = 0;
        while (true) {
            RecordCursorIterator<? extends FDBRecord<Message>> cursor;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version [[0],[0]])", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
            } else {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]])", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
            }
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBRecord<Message> record = cursor.next();
                MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
                assertEquals(0, simpleRecord.getNumValue2());
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(simpleRecord.getRecNo());
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals((simpleRecords.size() + 1) / 2, totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query that requires also filtering
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0 && rec.getNumValue3Indexed() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
        List<Long> receivedKeys = new ArrayList<>();
        FDBRecordVersion last = null;
        int totalSeen = 0;
        while (true) {
            RecordCursorIterator<? extends FDBRecord<Message>> cursor;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version [[0],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
            } else {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
            }
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBRecord<Message> record = cursor.next();
                MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
                assertEquals(0, simpleRecord.getNumValue2());
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(simpleRecord.getRecNo());
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals(simpleRecords.size() / 6, totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query that can't be satisfied with an index scan.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        // Preliminary query to get a read version.
        RecordQuery prelimQuery = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
        RecordQueryPlan prelimPlan = planner.plan(prelimQuery);
        FDBRecordVersion chosenVersion = recordStore.executeQuery(prelimPlan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asList().thenApply(list -> list.get(list.size() - 1).getVersion()).join();
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.version().greaterThan(chosenVersion)).setSort(field("num_value_3_indexed")).build();
        RecordQueryPlan plan = planner.plan(query);
        assertEquals("Index(MySimpleRecord$num_value_3_indexed <,>) | version GREATER_THAN " + chosenVersion.toString(), plan.toString());
        List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
        int last = -1;
        for (FDBQueriedRecord<Message> record : records) {
            MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
            assertThat(last, lessThanOrEqualTo(simpleRecord.getNumValue3Indexed()));
            assertTrue(record.hasVersion());
            assertThat(chosenVersion, lessThan(record.getVersion()));
            last = simpleRecord.getNumValue3Indexed();
        }
        assertEquals(simpleRecords.size() - 5, records.size());
    }
}

Example 74 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method assertMaxVersionWithFunction.

@SuppressWarnings("try")
private void assertMaxVersionWithFunction(int controlColumn, @Nonnull FDBRecordVersion recordVersion) {
    try (FDBRecordContext context = openContext(maxEverVersionWithFunctionHook)) {
        Index index = metaData.getIndex("max_ever_version_with_function");
        IndexEntry entry = new IndexEntry(index, Key.Evaluated.EMPTY, Key.Evaluated.concatenate(controlColumn, recordVersion));
        assertMaxVersionEntries(index, Collections.singletonList(entry));
    }
}

Example 75 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method enableRecordVersionsAfterTheFact.

@ParameterizedTest(name = "enableRecordVersionsAfterTheFact [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void enableRecordVersionsAfterTheFact(int testFormatVersion, boolean testSplitLongRecords) throws ExecutionException, InterruptedException {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(871L).setNumValue2(871).build();
    MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1415L).setNumValue2(1415).build();
    MySimpleRecord record3 = MySimpleRecord.newBuilder().setRecNo(3415L).setNumValue2(3415).build();
    Index globalCountIndex = new Index("globalCount", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
    RecordMetaDataHook origHook = metaDataBuilder -> {
        noVersionHook.apply(metaDataBuilder);
        metaDataBuilder.addIndex((RecordTypeBuilder) null, globalCountIndex);
    };
    try (FDBRecordContext context = openContext(origHook)) {
        assertFalse(metaData.isStoreRecordVersions());
        recordStore.saveRecord(record1);
        recordStore.saveRecord(record2);
        recordStore.saveRecord(record3, null, FDBRecordStoreBase.VersionstampSaveBehavior.WITH_VERSION);
        context.commit();
    }
    try (FDBRecordContext context = openContext(metaDataBuilder -> {
        origHook.apply(metaDataBuilder);
        metaDataBuilder.setStoreRecordVersions(true);
        functionVersionHook.apply(metaDataBuilder);
    })) {
        assertTrue(metaData.isStoreRecordVersions());
        FDBStoredRecord<Message> storedRecord1 = recordStore.loadRecord(Tuple.from(871L));
        assertNotNull(storedRecord1);
        assertEquals(record1, storedRecord1.getRecord());
        assertFalse(storedRecord1.hasVersion());
        FDBStoredRecord<Message> storedRecord2 = recordStore.loadRecord(Tuple.from(1415L));
        assertNotNull(storedRecord2);
        assertEquals(record2, storedRecord2.getRecord());
        assertFalse(storedRecord2.hasVersion());
        FDBStoredRecord<Message> storedRecord3 = recordStore.loadRecord(Tuple.from(3415L));
        assertNotNull(storedRecord3);
        assertEquals(record3, storedRecord3.getRecord());
        assertTrue(storedRecord3.hasVersion());
        RecordCursor<IndexEntry> cursor = recordStore.scanIndex(metaData.getIndex("MySimpleRecord$maybeVersion"), IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN);
        assertEquals(Arrays.asList(Tuple.from(null, 1415L), Tuple.from(FDBRecordVersion.MIN_VERSION.toVersionstamp(), 871L), Tuple.from(storedRecord3.getVersion().toVersionstamp(), 3415L)), cursor.map(IndexEntry::getKey).asList().get());
    }
}