Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 56 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBOrQueryToUnionTest method testOrQuery6.

/**
 * Verify that a complex query with an OR of an AND produces a union plan if appropriate indexes are defined.
 * In particular, verify that it can use the last field of an index and does not require primary key ordering
 * compatibility.
 */
@DualPlannerTest
void testOrQuery6() throws Exception {
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", new Index("str_value_3_index", "str_value_indexed", "num_value_3_indexed"));
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.and(Query.field("str_value_indexed").equalsValue("even"), Query.field("num_value_3_indexed").greaterThan(3)), Query.field("num_value_3_indexed").lessThan(1))).build();
    // Index(str_value_3_index ([even, 3],[even]]) ∪[Field { 'num_value_3_indexed' None}, Field { 'rec_no' None}] Index(MySimpleRecord$num_value_3_indexed ([null],[1]))
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof CascadesPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(unionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("([even, 3],[even]]"))))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("([null],[1])")))))).where(comparisonKey(concat(Key.Expressions.field("num_value_3_indexed"), primaryKey("MySimpleRecord")))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-835124758, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(778876973, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1061354639, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = unionPlan(indexPlan().where(indexName("str_value_3_index")).and(scanComparisons(range("([even, 3],[even]]"))), indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")).and(scanComparisons(range("([null],[1])")))).where(comparisonKey(concat(Key.Expressions.field("num_value_3_indexed"), primaryKey("MySimpleRecord"))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(1721396731, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1374663850, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1092186184, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue((myrec.getStrValueIndexed().equals("even") && myrec.getNumValue3Indexed() > 3) || myrec.getNumValue3Indexed() < 1);
                i++;
            }
        }
        assertEquals(20 + 10, i);
        assertDiscardedNone(context);
    }
}

Example 57 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class TextIndexTest method queryComplexDocumentsWithAdditionalFilters.

@Test
public void queryComplexDocumentsWithAdditionalFilters() throws Exception {
    final List<String> textSamples = Arrays.asList(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH, TextSamples.GERMAN, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.YIDDISH, "Napoleon and the Duke of Wellington met in Waterloo in 1815.");
    final List<ComplexDocument> documents = IntStream.range(0, textSamples.size()).mapToObj(i -> ComplexDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).addTag("3:" + (i % 3)).setScore(i).build()).collect(Collectors.toList());
    final Index rankIndex = new Index("Complex$rank(score)", field("score").groupBy(field("group")), IndexTypes.RANK);
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> {
            final RecordTypeBuilder complexDocRecordType = metaDataBuilder.getRecordType(COMPLEX_DOC);
            metaDataBuilder.addIndex(complexDocRecordType, COMPLEX_TEXT_BY_GROUP);
            metaDataBuilder.addIndex(complexDocRecordType, rankIndex);
        });
        documents.forEach(recordStore::saveRecord);
        assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.field("tag").oneOfThem().equalsValue("3:2"), 1, 758136568));
        assertEquals(Arrays.asList(Tuple.from(1L, 1L), Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.field("text").text().containsPrefix("continu"), 1, -1043653062));
        assertEquals(Collections.singletonList(Tuple.from(1L, 7L)), queryComplexDocumentsWithIndex(Query.field("text").text().contains("napoleon"), Query.and(Query.field("text").text().containsPrefix("th"), Query.field("text").text().contains("waterloo")), 1, -754900112));
        assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.not(Query.field("tag").oneOfThem().equalsValue("3:2")), 1, 758136569));
        assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.not(Query.field("tag").oneOfThem().equalsValue("3:2")), 1, 758136569));
        assertEquals(Arrays.asList(Tuple.from(0L, 0L), Tuple.from(0L, 6L)), queryComplexDocumentsWithOr((OrComponent) Query.or(Query.field("text").text().containsAll("unit named after"), Query.field("text").text().containsPhrase("אן ארמיי און פלאט")), 0, -1558384887));
        assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.or(Query.field("tag").oneOfThem().equalsValue("3:2"), Query.field("tag").oneOfThem().equalsValue("3:0")), true, 1, -27568755));
        assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.rank(field("score").groupBy(field("group"))).lessThan(2L), true, 1, -2132208833));
        assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("fear pass love", true), Query.field("tag").oneOfThem().equalsValue("3:2"), true, 1, -419325379));
        assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("fear pass love", false), Query.field("tag").oneOfThem().equalsValue("3:2"), false, 1, -1902024530));
        assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("fear pass love"), Query.rank(field("score").groupBy(field("group"))).lessThan(2L), true, 1, 669157421));
        commit(context);
    }
}

Example 58 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class TextIndexTest method saveSimpleDocuments.

@Test
public void saveSimpleDocuments() throws Exception {
    final SimpleDocument simpleDocument = SimpleDocument.newBuilder().setDocId(1066L).setText("This is a simple document. There isn't much going on here, if I'm honest.").setGroup(0).build();
    final SimpleDocument buffaloDocument = SimpleDocument.newBuilder().setDocId(1415L).setText("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo Buffalo buffalo buffalo.").setGroup(1).build();
    final SimpleDocument shakespeareDocument = SimpleDocument.newBuilder().setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).setGroup(2).build();
    final SimpleDocument noTextDocument = SimpleDocument.newBuilder().setDocId(0L).setGroup(0).build();
    final SimpleDocument emptyDocument = SimpleDocument.newBuilder().setDocId(1L).setGroup(1).setText("").build();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        Index index = recordStore.getRecordMetaData().getIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
        recordStore.saveRecord(simpleDocument);
        final int firstKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(simpleDocument.getText().split(" ").length, firstKeys);
        final int firstKeyBytesWritten = getSaveIndexKeyBytes(recordStore);
        final int firstValueBytesWritten = getSaveIndexValueBytes(recordStore);
        List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(4))), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(buffaloDocument);
        final int secondKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(1, secondKeys);
        entryList = scanMapEntries(recordStore, index, Tuple.from("buffalo"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1415L), IntStream.range(0, 11).boxed().collect(Collectors.toList()))), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(shakespeareDocument);
        final int thirdKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(82, thirdKeys);
        final int thirdBytesWritten = getSaveIndexKeyBytes(recordStore) + getSaveIndexValueBytes(recordStore);
        entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1623L), Arrays.asList(57, 72))), entryList);
        entryList = toMapEntries(scanIndex(recordStore, index, TupleRange.prefixedBy("h")), null);
        assertEquals(Arrays.asList(entryOf(Tuple.from("hands", 1623), Collections.singletonList(26)), entryOf(Tuple.from("here", 1066), Collections.singletonList(10)), entryOf(Tuple.from("here", 1623), Collections.singletonList(101)), entryOf(Tuple.from("honest", 1066), Collections.singletonList(13)), entryOf(Tuple.from("hours", 1623), Collections.singletonList(87)), entryOf(Tuple.from("households", 1623), Collections.singletonList(1))), entryList);
        List<Message> recordList = recordStore.scanIndexRecords(index.getName(), BY_TEXT_TOKEN, TupleRange.prefixedBy("h"), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
        assertEquals(Arrays.asList(shakespeareDocument, simpleDocument, shakespeareDocument, simpleDocument, shakespeareDocument, shakespeareDocument), recordList);
        resetTimer(recordStore);
        recordStore.saveRecord(noTextDocument);
        assertEquals(0, getSaveIndexKeyCount(recordStore));
        assertEquals(0, getLoadIndexKeyCount(recordStore));
        resetTimer(recordStore);
        recordStore.saveRecord(emptyDocument);
        assertEquals(0, getSaveIndexKeyCount(recordStore));
        assertEquals(0, getLoadIndexKeyCount(recordStore));
        resetTimer(recordStore);
        recordStore.deleteRecord(Tuple.from(1623L));
        // all deleted but four overlaps with first record
        assertEquals(thirdKeys - 4, getDeleteIndexKeyCount(recordStore));
        // four keys of overlap overwritten
        assertEquals(4, getSaveIndexKeyCount(recordStore));
        assertThat(getDeleteIndexKeyBytes(recordStore) + getDeleteIndexValueBytes(recordStore), allOf(greaterThan(thirdKeys - 1), lessThan(thirdBytesWritten)));
        entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
        assertEquals(Collections.emptyList(), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(simpleDocument.toBuilder().setDocId(1707L).build());
        assertEquals(firstKeys * 2, getLoadIndexKeyCount(recordStore));
        assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
        // should overwrite all the same keys
        assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
        final int seventhValueBytesWritten = getSaveIndexValueBytes(recordStore);
        // contains same info as first value bytes + extra keys, but not key prefixes
        assertThat(seventhValueBytesWritten, allOf(greaterThan(firstValueBytesWritten), lessThan(firstKeyBytesWritten + firstValueBytesWritten)));
        entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(4)), entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
        resetTimer(recordStore);
        recordStore.deleteRecord(Tuple.from(1066L));
        assertEquals(firstKeys, getLoadIndexKeyCount(recordStore));
        // each of the original keys are deleted
        assertEquals(firstKeys, getDeleteIndexKeyCount(recordStore));
        assertEquals(firstKeyBytesWritten, getDeleteIndexKeyBytes(recordStore));
        assertEquals(firstValueBytesWritten + seventhValueBytesWritten, getDeleteIndexValueBytes(recordStore));
        // a new set of keys are all written
        assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
        // they should have the same size (though their contents are different)
        assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
        assertEquals(firstValueBytesWritten, getSaveIndexValueBytes(recordStore));
        entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
        commit(context);
    }
}

Example 59 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method minMaxValue.

@Test
public void minMaxValue() throws Exception {
    final FieldKeyExpression numValue2 = field("num_value_2");
    final FieldKeyExpression numValue3 = field("num_value_3_indexed");
    final ThenKeyExpression compound = concat(numValue2, numValue3);
    final GroupingKeyExpression grouped = numValue3.groupBy(numValue2);
    final RecordMetaDataHook hook = md -> {
        RecordTypeBuilder type = md.getRecordType("MySimpleRecord");
        md.addIndex(type, new Index("compound", compound, IndexTypes.VALUE));
    };
    final IndexAggregateFunction minOverall = new IndexAggregateFunction(FunctionNames.MIN, numValue3, null);
    final IndexAggregateFunction maxOverall = new IndexAggregateFunction(FunctionNames.MAX, numValue3, null);
    final IndexAggregateFunction minByKey = new IndexAggregateFunction(FunctionNames.MIN, grouped, null);
    final IndexAggregateFunction maxByKey = new IndexAggregateFunction(FunctionNames.MAX, grouped, null);
    List<String> types = Collections.singletonList("MySimpleRecord");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertNull(recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        assertNull(recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join());
        for (int i = 0; i < 100; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setNumValue2(i % 5);
            recBuilder.setNumValue3Indexed(i + 1000);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        assertEquals(1000, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(1099, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(1001, recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(1096, recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteRecord(Tuple.from(0));
        recordStore.deleteRecord(Tuple.from(99));
        assertEquals(1001, recordStore.evaluateAggregateFunction(types, minOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(1098, recordStore.evaluateAggregateFunction(types, maxOverall, Key.Evaluated.EMPTY, IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(1001, recordStore.evaluateAggregateFunction(types, minByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        assertEquals(1096, recordStore.evaluateAggregateFunction(types, maxByKey, Key.Evaluated.scalar(1), IsolationLevel.SNAPSHOT).join().getLong(0));
        commit(context);
    }
}

Example 60 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method indexesToBuild.

@Test
public void indexesToBuild() throws Exception {
    final String indexName = "MySimpleRecord$str_value_indexed";
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final Index index = recordStore.getRecordMetaData().getIndex(indexName);
        assertTrue(recordStore.markIndexDisabled(index).get(), "index should not have been disabled initially");
        assertTrue(recordStore.getIndexesToBuild().containsKey(index), "index should have been included in indexes to build");
        recordStore.rebuildIndex(index).get();
        assertEquals(Collections.emptyMap(), recordStore.getIndexesToBuild());
        commit(context);
    }
}