Examples with TextSamples com.apple.foundationdb.record.provider.common.text.TextSamples used on opensource projects

Example 6 with TextSamples

use of com.apple.foundationdb.record.provider.common.text.TextSamples in project fdb-record-layer by FoundationDB.

the class TextIndexTest method queryMapDocuments.

@Test
public void queryMapDocuments() throws Exception {
    final List<String> textSamples = Arrays.asList(TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.ANGSTROM, TextSamples.AETHELRED, TextSamples.FRENCH);
    final List<MapDocument> documents = IntStream.range(0, textSamples.size() / 2).mapToObj(i -> MapDocument.newBuilder().setDocId(i).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(textSamples.get(i * 2)).build()).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue(textSamples.get(i * 2 + 1)).build()).setGroup(i % 2).build()).collect(Collectors.toList());
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_INDEX));
        documents.forEach(recordStore::saveRecord);
        assertEquals(Collections.singletonList(2L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsAny("king unknown_token"), 1059912699, true));
        assertEquals(Arrays.asList(0L, 1L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsPhrase("civil blood makes civil hands unclean"), 1085034960, true));
        assertEquals(Collections.emptyList(), queryMapDocumentsWithIndex("b", Query.field("value").text().containsPhrase("civil blood makes civil hands unclean"), 1085034991, true));
        assertEquals(Arrays.asList(1L, 2L), queryMapDocumentsWithIndex("b", Query.field("value").text().containsPrefix("na"), 1125182095, true));
        assertEquals(Arrays.asList(0L, 1L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsAllPrefixes("civ mut ha"), 0, false));
        assertEquals(Arrays.asList(1L, 2L), queryMapDocumentsWithIndex("b", Query.field("value").text().containsAnyPrefix("civ mut na"), 0, true));
        RecordQuery queryWithAdditionalFilter = RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(Query.and(Query.field("group").equalsValue(0L), Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("b"), Query.field("value").text().containsAny("anders king"))))).build();
        RecordQueryPlan planWithAdditionalFilter = recordStore.planQuery(queryWithAdditionalFilter);
        assertThat(planWithAdditionalFilter, filter(Query.field("group").equalsValue(0L), descendant(textIndexScan(anything()))));
        List<Long> queryResults = recordStore.executeQuery(planWithAdditionalFilter).map(FDBQueriedRecord::getPrimaryKey).map(tuple -> tuple.getLong(0)).asList().join();
        assertEquals(Collections.singletonList(0L), queryResults);
        queryWithAdditionalFilter = RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(Query.or(Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("a"), Query.field("value").text().containsPhrase("bury their parents strife"))), Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("b"), Query.field("value").text().containsPrefix("th"))))).build();
        planWithAdditionalFilter = recordStore.planQuery(queryWithAdditionalFilter);
        assertThat(planWithAdditionalFilter, primaryKeyDistinct(unorderedUnion(descendant(textIndexScan(indexName(equalTo(MAP_ON_VALUE_INDEX.getName())))), descendant(textIndexScan(indexName(equalTo(MAP_ON_VALUE_INDEX.getName())))))));
        queryResults = recordStore.executeQuery(planWithAdditionalFilter).map(FDBQueriedRecord::getPrimaryKey).map(tuple -> tuple.getLong(0)).asList().join();
        assertEquals(3, queryResults.size());
        assertEquals(ImmutableSet.of(0L, 1L, 2L), ImmutableSet.copyOf(queryResults));
        // Planner bug that can happen with certain malformed queries. This plan actually
        // returns records where the key and the value match in the same entry, but it is
        // asking for all records where *any* entry has a key matching "a" and *any* entry
        // has a value matching the text predicate. In reality, this is probably a sign
        // the user didn't input their query correctly, but it requires more work from the
        // planner not to plan this kind of query.
        // FIXME: Full Text: The Planner doesn't always correctly handle ands with nesteds (https://github.com/FoundationDB/fdb-record-layer/issues/53)
        final QueryComponent malformedMapFilter = Query.and(Query.field("entry").oneOfThem().matches(Query.field("key").equalsValue("a")), Query.field("entry").oneOfThem().matches(Query.field("value").text().containsAll("civil hands unclean")));
        RecordQueryPlan malformedMapPlan = planner.plan(RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(malformedMapFilter).build());
        assertThat(malformedMapPlan, descendant(textIndexScan(allOf(indexName(MAP_ON_VALUE_INDEX.getName()), groupingBounds(allOf(notNullValue(), hasTupleString("[[a],[a]]"))), textComparison(equalTo(new Comparisons.TextComparison(Comparisons.Type.TEXT_CONTAINS_ALL, "civil hands unclean", null, DefaultTextTokenizer.NAME)))))));
        commit(context);
    }
}

Example 7 with TextSamples

use of com.apple.foundationdb.record.provider.common.text.TextSamples in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreByteLimitTest method queryWithWideOrOfFullTextPrefixPredicates.

/**
 * Queries with an OR of {@link com.apple.foundationdb.record.query.expressions.Text#containsPrefix(String)}
 * predicates get planned as {@link com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan}s,
 * which have unusual semantics where results are returned in an undefined order as soon as any child has one.
 * Therefore, the assertions made in {@link #assertPlanLimitsWithCorrectExecution(List, FDBRecordContext, RecordQueryPlan)}
 * are far too strong for plans like this. Instead, we make very weak assertions that the byte scan limit does
 * <em>something</em>.
 */
@ParameterizedTest
@MethodSource("complexTextQueries")
public void queryWithWideOrOfFullTextPrefixPredicates(@Nonnull RecordQuery query, int numPredicates) throws Exception {
    deleteSimpleRecords();
    final List<String> textSamples = ImmutableList.of(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH, TextSamples.KOREAN);
    RecordMetaDataHook indexHook = metaDataBuilder -> metaDataBuilder.addIndex(metaDataBuilder.getRecordType(SIMPLE_DOC), SIMPLE_TEXT_PREFIX);
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        for (int i = 0; i < textSamples.size(); i++) {
            recordStore.saveRecord(TestRecordsTextProto.SimpleDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).build());
        }
        commit(context);
    }
    setupPlanner(null);
    RecordQueryPlan plan = planner.plan(query);
    assertThat(plan, descendant(unorderedUnion(Collections.nCopies(numPredicates, any(RecordQueryPlan.class)))));
    long totalBytes;
    Set<Long> noLimitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context, indexHook);
        context.getTimer().reset();
        RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, null, ExecuteProperties.SERIAL_EXECUTE);
        RecordCursorResult<FDBQueriedRecord<Message>> result;
        do {
            result = cursor.onNext().get();
            if (result.hasNext()) {
                TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                record.mergeFrom(result.get().getRecord());
                noLimitRecordIds.add(record.getDocId());
            }
        } while (result.hasNext());
        totalBytes = byteCounter.getBytesScanned(context);
    }
    Set<Long> limitRecordIds = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openTextRecordStore(context);
        ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(0);
        byte[] continuation = null;
        do {
            context.getTimer().reset();
            RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, continuation, executeProperties.build());
            RecordCursorResult<FDBQueriedRecord<Message>> result;
            do {
                result = cursor.onNext().get();
                if (result.hasNext()) {
                    TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
                    record.mergeFrom(result.get().getRecord());
                    limitRecordIds.add(record.getDocId());
                }
            } while (result.hasNext());
            assertThat(byteCounter.getBytesScanned(context), lessThan(totalBytes));
            continuation = result.getContinuation().toBytes();
            if (continuation != null) {
                assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
            }
        } while (continuation != null);
        assertEquals(noLimitRecordIds, limitRecordIds);
    }
}