Search in sources :

Example 1 with FDBIndexedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.

the class FDBCrossRecordQueryTest method testMultiRecordTypeIndexScan.

/**
 * Verify that multi-type record queries can scan multi-type indexes.
 * Verify that single-type record queries can scan multi-type indexes with a type filter.
 */
@Test
public void testMultiRecordTypeIndexScan() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openUnionRecordStore(context);
        saveSimpleRecord(100, "first", 1);
        saveSimpleRecord(110, "second", 2);
        saveSimpleRecord2("third", 3);
        saveSimpleRecord2("fourth", 4);
        saveSimpleRecord(80, "fifth", 5);
        saveSimpleRecord2("sixth", 6);
        saveSimpleRecord2("seventh", 7);
        saveSimpleRecord(60, "seventh", 7);
        saveSimpleRecord2("seventh again", 7);
        saveSimpleRecord3("t3 second", 2);
        saveSimpleRecord3("t3 sixth", 6);
        saveSimpleRecord3("t3 seventh", 7);
        commit(context);
    }
    List<String> names = new ArrayList<>();
    List<Integer> etags = new ArrayList<>();
    try (FDBRecordContext context = openContext()) {
        openUnionRecordStore(context);
        try (RecordCursorIterator<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("partial_versions", IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asIterator()) {
            while (cursor.hasNext()) {
                final Message record = cursor.next().getRecord();
                names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
            }
        }
        assertDiscardedNone(context);
    }
    assertEquals(Arrays.asList("first", "second", "third", "fourth", "fifth", "sixth"), names.subList(0, 6));
    assertThat(names.subList(6, 9), containsInAnyOrder("seventh", "seventh", "seventh again"));
    assertEquals(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 7, 7), etags);
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MySimpleRecord", "MySimpleRecord2")).setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(partial_versions [[7],[7]])
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, indexScan(allOf(indexName("partial_versions"), bounds(hasTupleString("[[7],[7]]")))));
        assertEquals(-501898489, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-1416119651, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1071937908, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedNone(context);
        }
        assertThat(names, containsInAnyOrder("seventh", "seventh", "seventh again"));
        assertEquals(Arrays.asList(7, 7, 7), etags);
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord2").setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(partial_versions [[7],[7]]) | [MySimpleRecord2]
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, typeFilter(contains("MySimpleRecord2"), indexScan(allOf(indexName("partial_versions"), bounds(hasTupleString("[[7],[7]]"))))));
        assertEquals(-1724404567, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1091241424, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1124026431, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedAtMost(1, context);
        }
        assertThat(names, containsInAnyOrder("seventh", "seventh again"));
        assertEquals(Arrays.asList(7, 7), etags);
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord3").setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(versions [[7],[7]]) | [MySimpleRecord3]
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, typeFilter(contains("MySimpleRecord3"), indexScan(allOf(indexName("versions"), bounds(hasTupleString("[[7],[7]]"))))));
        assertEquals(-1908726868, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(168009743, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-2047258112, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            clearStoreCounter(context);
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedAtMost(3, context);
        }
        assertEquals(Arrays.asList("t3 seventh"), names);
        assertEquals(Arrays.asList(7), etags);
    }
    {
        RecordQuery query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MySimpleRecord2", "MySimpleRecord3")).setFilter(Query.field("etag").equalsValue(7)).build();
        // Index(versions [[7],[7]]) | [MySimpleRecord2, MySimpleRecord3]
        RecordQueryPlan plan = planner.plan(query);
        MatcherAssert.assertThat(plan, typeFilter(containsInAnyOrder("MySimpleRecord2", "MySimpleRecord3"), indexScan(allOf(indexName("versions"), bounds(hasTupleString("[[7],[7]]"))))));
        assertEquals(-1151709653, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(925026958, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1290240897, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        names.clear();
        etags.clear();
        try (FDBRecordContext context = openContext()) {
            clearStoreCounter(context);
            openUnionRecordStore(context);
            try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
                while (cursor.hasNext()) {
                    final Message record = cursor.next().getRecord();
                    names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
                    etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
                }
            }
            assertDiscardedAtMost(1, context);
        }
        assertThat(names, containsInAnyOrder("seventh", "seventh again", "t3 seventh"));
        assertEquals(Arrays.asList(7, 7, 7), etags);
    }
}
Also used : RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ArrayList(java.util.ArrayList) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Test(org.junit.jupiter.api.Test)

Example 2 with FDBIndexedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.

the class TextIndexTest method saveSimpleDocuments.

@Test
public void saveSimpleDocuments() throws Exception {
    final SimpleDocument simpleDocument = SimpleDocument.newBuilder().setDocId(1066L).setText("This is a simple document. There isn't much going on here, if I'm honest.").setGroup(0).build();
    final SimpleDocument buffaloDocument = SimpleDocument.newBuilder().setDocId(1415L).setText("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo Buffalo buffalo buffalo.").setGroup(1).build();
    final SimpleDocument shakespeareDocument = SimpleDocument.newBuilder().setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).setGroup(2).build();
    final SimpleDocument noTextDocument = SimpleDocument.newBuilder().setDocId(0L).setGroup(0).build();
    final SimpleDocument emptyDocument = SimpleDocument.newBuilder().setDocId(1L).setGroup(1).setText("").build();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        Index index = recordStore.getRecordMetaData().getIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
        recordStore.saveRecord(simpleDocument);
        final int firstKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(simpleDocument.getText().split(" ").length, firstKeys);
        final int firstKeyBytesWritten = getSaveIndexKeyBytes(recordStore);
        final int firstValueBytesWritten = getSaveIndexValueBytes(recordStore);
        List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(4))), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(buffaloDocument);
        final int secondKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(1, secondKeys);
        entryList = scanMapEntries(recordStore, index, Tuple.from("buffalo"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1415L), IntStream.range(0, 11).boxed().collect(Collectors.toList()))), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(shakespeareDocument);
        final int thirdKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(82, thirdKeys);
        final int thirdBytesWritten = getSaveIndexKeyBytes(recordStore) + getSaveIndexValueBytes(recordStore);
        entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1623L), Arrays.asList(57, 72))), entryList);
        entryList = toMapEntries(scanIndex(recordStore, index, TupleRange.prefixedBy("h")), null);
        assertEquals(Arrays.asList(entryOf(Tuple.from("hands", 1623), Collections.singletonList(26)), entryOf(Tuple.from("here", 1066), Collections.singletonList(10)), entryOf(Tuple.from("here", 1623), Collections.singletonList(101)), entryOf(Tuple.from("honest", 1066), Collections.singletonList(13)), entryOf(Tuple.from("hours", 1623), Collections.singletonList(87)), entryOf(Tuple.from("households", 1623), Collections.singletonList(1))), entryList);
        List<Message> recordList = recordStore.scanIndexRecords(index.getName(), BY_TEXT_TOKEN, TupleRange.prefixedBy("h"), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
        assertEquals(Arrays.asList(shakespeareDocument, simpleDocument, shakespeareDocument, simpleDocument, shakespeareDocument, shakespeareDocument), recordList);
        resetTimer(recordStore);
        recordStore.saveRecord(noTextDocument);
        assertEquals(0, getSaveIndexKeyCount(recordStore));
        assertEquals(0, getLoadIndexKeyCount(recordStore));
        resetTimer(recordStore);
        recordStore.saveRecord(emptyDocument);
        assertEquals(0, getSaveIndexKeyCount(recordStore));
        assertEquals(0, getLoadIndexKeyCount(recordStore));
        resetTimer(recordStore);
        recordStore.deleteRecord(Tuple.from(1623L));
        // all deleted but four overlaps with first record
        assertEquals(thirdKeys - 4, getDeleteIndexKeyCount(recordStore));
        // four keys of overlap overwritten
        assertEquals(4, getSaveIndexKeyCount(recordStore));
        assertThat(getDeleteIndexKeyBytes(recordStore) + getDeleteIndexValueBytes(recordStore), allOf(greaterThan(thirdKeys - 1), lessThan(thirdBytesWritten)));
        entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
        assertEquals(Collections.emptyList(), entryList);
        resetTimer(recordStore);
        recordStore.saveRecord(simpleDocument.toBuilder().setDocId(1707L).build());
        assertEquals(firstKeys * 2, getLoadIndexKeyCount(recordStore));
        assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
        // should overwrite all the same keys
        assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
        final int seventhValueBytesWritten = getSaveIndexValueBytes(recordStore);
        // contains same info as first value bytes + extra keys, but not key prefixes
        assertThat(seventhValueBytesWritten, allOf(greaterThan(firstValueBytesWritten), lessThan(firstKeyBytesWritten + firstValueBytesWritten)));
        entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(4)), entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
        resetTimer(recordStore);
        recordStore.deleteRecord(Tuple.from(1066L));
        assertEquals(firstKeys, getLoadIndexKeyCount(recordStore));
        // each of the original keys are deleted
        assertEquals(firstKeys, getDeleteIndexKeyCount(recordStore));
        assertEquals(firstKeyBytesWritten, getDeleteIndexKeyBytes(recordStore));
        assertEquals(firstValueBytesWritten + seventhValueBytesWritten, getDeleteIndexValueBytes(recordStore));
        // a new set of keys are all written
        assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
        // they should have the same size (though their contents are different)
        assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
        assertEquals(firstValueBytesWritten, getSaveIndexValueBytes(recordStore));
        entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
        commit(context);
    }
}
Also used : FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) BunchedMapScanEntry(com.apple.foundationdb.map.BunchedMapScanEntry) IndexEntry(com.apple.foundationdb.record.IndexEntry) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) SimpleDocument(com.apple.foundationdb.record.TestRecordsTextProto.SimpleDocument) Index(com.apple.foundationdb.record.metadata.Index) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 3 with FDBIndexedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.

the class TextIndexTest method saveComplexDocuments.

@Test
public void saveComplexDocuments() throws Exception {
    ComplexDocument complexDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1066L).setText("Very complex. Not to be trifled with.").build();
    ComplexDocument shakespeareDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).addTag("a").addTag("b").build();
    ComplexDocument yiddishDocument = ComplexDocument.newBuilder().setGroup(1).setDocId(1944L).setText(TextSamples.YIDDISH).addTag("c").build();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_TEXT_BY_GROUP));
        recordStore.saveRecord(complexDocument);
        int firstKeys = getSaveIndexKeyCount(recordStore);
        assertEquals(complexDocument.getText().split(" ").length, firstKeys);
        recordStore.saveRecord(shakespeareDocument);
        int secondKeys = getSaveIndexKeyCount(recordStore) - firstKeys;
        assertEquals(82, secondKeys);
        recordStore.saveRecord(yiddishDocument);
        int thirdKeys = getSaveIndexKeyCount(recordStore) - secondKeys - firstKeys;
        assertEquals(9, thirdKeys);
        List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, COMPLEX_TEXT_BY_GROUP, Tuple.from(0L, "to"));
        assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(3)), entryOf(Tuple.from(1623L), Arrays.asList(18, 108))), entryList);
        List<Message> recordList = recordStore.scanIndexRecords(COMPLEX_TEXT_BY_GROUP.getName(), BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(0L, "to")), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
        assertEquals(Arrays.asList(complexDocument, shakespeareDocument), recordList);
        entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.prefixedBy("א").prepend(Tuple.from(1L))), null);
        assertEquals(Arrays.asList(entryOf(Tuple.from(1L, "א", 1944L), Arrays.asList(0, 3)), entryOf(Tuple.from(1L, "און", 1944L), Collections.singletonList(8)), entryOf(Tuple.from(1L, "איז", 1944L), Collections.singletonList(2)), entryOf(Tuple.from(1L, "אן", 1944L), Collections.singletonList(6)), entryOf(Tuple.from(1L, "ארמיי", 1944L), Collections.singletonList(7))), entryList);
        // Read the whole store and make sure the values come back in a somewhat sensible way
        entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.ALL), null);
        assertEquals(firstKeys + secondKeys + thirdKeys, entryList.size());
        int i = 0;
        String last = null;
        for (Map.Entry<Tuple, List<Integer>> entry : entryList) {
            assertEquals(3, entry.getKey().size());
            if (i < firstKeys + secondKeys) {
                assertEquals(0L, entry.getKey().getLong(0));
                assertThat(entry.getKey().getLong(2), anyOf(is(1066L), is(1623L)));
            } else {
                assertEquals(1L, entry.getKey().getLong(0));
                assertEquals(1944L, entry.getKey().getLong(2));
                if (i == firstKeys + secondKeys) {
                    last = null;
                }
            }
            if (last != null) {
                assertThat(entry.getKey().getString(1), greaterThanOrEqualTo(last));
            }
            last = entry.getKey().getString(1);
            i++;
        }
        commit(context);
    }
}
Also used : FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Message(com.google.protobuf.Message) Matchers.containsString(org.hamcrest.Matchers.containsString) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) ComplexDocument(com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument) BunchedMapScanEntry(com.apple.foundationdb.map.BunchedMapScanEntry) IndexEntry(com.apple.foundationdb.record.IndexEntry) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ArrayList(java.util.ArrayList) List(java.util.List) Map(java.util.Map) BunchedMap(com.apple.foundationdb.map.BunchedMap) ImmutableMap(com.google.common.collect.ImmutableMap) Tuple(com.apple.foundationdb.tuple.Tuple) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 4 with FDBIndexedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.

the class RankIndexTest method checkRankScan.

@Test
public void checkRankScan() throws Exception {
    TupleRange range = new TupleRange(Tuple.from(0L), Tuple.from(2L), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        int i = 0;
        try (RecordCursorIterator<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("BasicRankedRecord$score", IndexScanType.BY_RANK, range, null, ScanProperties.FORWARD_SCAN).asIterator()) {
            while (cursor.hasNext()) {
                FDBIndexedRecord<Message> rec = cursor.next();
                TestRecordsRankProto.BasicRankedRecord.Builder myrec = TestRecordsRankProto.BasicRankedRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertTrue(myrec.getScore() < 200);
                i++;
            }
        }
        assertEquals(2, i);
    }
    range = TupleRange.allOf(Tuple.from("M", 0L));
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        try (RecordCursor<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("rank_by_gender", IndexScanType.BY_RANK, range, null, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(1).build()))) {
            FDBIndexedRecord<Message> rec = cursor.getNext().get();
            TestRecordsRankProto.BasicRankedRecord.Builder myrec = TestRecordsRankProto.BasicRankedRecord.newBuilder();
            myrec.mergeFrom(rec.getRecord());
            assertEquals("hector", myrec.getName());
            assertEquals(75, myrec.getScore());
        }
    }
}
Also used : FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) Message(com.google.protobuf.Message) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanProperties(com.apple.foundationdb.record.ScanProperties) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test)

Example 5 with FDBIndexedRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.

the class GeophileQueryTest method testJoin.

@Test
@Tag(Tags.Slow)
public // So really this just shows that join results aren't wildly off.
void testJoin() throws Exception {
    final RecordMetaDataHook hook = md -> {
        md.setSplitLongRecords(true);
        md.addIndex("City", CITY_LOCATION_INDEX);
        md.addIndex("Country", COUNTRY_SHAPE_INDEX);
    };
    loadCities(hook, 500000);
    loadCountries(hook, 500000);
    loadCountryShapes(hook);
    final GeophileSpatialIndexJoinPlan plan = new GeophileSpatialIndexJoinPlan("City$location", ScanComparisons.EMPTY, "Country$shape", ScanComparisons.EMPTY);
    int[] stats = new int[4];
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<Pair<FDBIndexedRecord<Message>, FDBIndexedRecord<Message>>> recordCursor = plan.execute(recordStore, EvaluationContext.EMPTY);
        final GeometryFactory geometryFactory = new GeometryFactory();
        final GeoJsonReader geoJsonReader = new GeoJsonReader(geometryFactory);
        recordCursor.forEach(pair -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(pair.getLeft().getRecord());
            TestRecordsGeoProto.Country.Builder countryBuilder = TestRecordsGeoProto.Country.newBuilder().mergeFrom(pair.getRight().getRecord());
            Point cityLocation = geometryFactory.createPoint(new Coordinate(cityBuilder.getLocation().getLatitude(), cityBuilder.getLocation().getLongitude()));
            Geometry countryShape;
            try {
                countryShape = GeophileSpatial.swapLatLong(geoJsonReader.read(countryBuilder.getShape()));
            } catch (ParseException ex) {
                throw new RuntimeException(ex);
            }
            // A real join would need to do this because of Z-order join false positives.
            boolean contained;
            try {
                contained = countryShape.contains(cityLocation);
            } catch (TopologyException ex) {
                stats[3]++;
                return;
            }
            if (!contained) {
                stats[2]++;
            } else if (!countryBuilder.getCode().equals(cityBuilder.getCountry())) {
                LOGGER.warn(String.format("Code does not match: %s for %s <> %s for %s", countryBuilder.getCode(), countryBuilder.getName(), cityBuilder.getCountry(), cityBuilder.getName()));
                stats[1]++;
            } else {
                LOGGER.debug(String.format("%s: %s", countryBuilder.getName(), cityBuilder.getName()));
                stats[0]++;
            }
        }).join();
        commit(context);
    } catch (CompletionException ex) {
        // transaction_too_old
        assertThat(Throwables.getRootCause(ex), allOf(Matchers.instanceOf(FDBException.class), hasProperty("code", equalTo(1007))));
    } finally {
        LOGGER.info(String.format("match = %d, no match = %d, no overlap = %d, invalid geometry = %d", stats[0], stats[1], stats[2], stats[3]));
    }
}
Also used : RecordQueryFilterPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryFilterPlan) LoggerFactory(org.slf4j.LoggerFactory) Coordinate(org.locationtech.jts.geom.Coordinate) Bindings(com.apple.foundationdb.record.Bindings) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Tuple(com.apple.foundationdb.tuple.Tuple) Pair(org.apache.commons.lang3.tuple.Pair) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) RecordQueryScanPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) Query(com.apple.foundationdb.record.query.expressions.Query) DoubleValueOrParameter(com.apple.foundationdb.record.spatial.common.DoubleValueOrParameter) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) Point(org.locationtech.jts.geom.Point) CompletionException(java.util.concurrent.CompletionException) Expressions.keyWithValue(com.apple.foundationdb.record.metadata.Key.Expressions.keyWithValue) Test(org.junit.jupiter.api.Test) GeoJsonReader(org.locationtech.jts.io.geojson.GeoJsonReader) TopologyException(org.locationtech.jts.geom.TopologyException) ParseException(org.locationtech.jts.io.ParseException) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Geometry(org.locationtech.jts.geom.Geometry) Expressions.function(com.apple.foundationdb.record.metadata.Key.Expressions.function) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) RecordQueryTypeFilterPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan) GeoPointWithinDistanceComponent(com.apple.foundationdb.record.spatial.common.GeoPointWithinDistanceComponent) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) FDBRecordStoreQueryTestBase(com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) HashSet(java.util.HashSet) Matchers.hasProperty(org.hamcrest.Matchers.hasProperty) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) GeometryFactory(org.locationtech.jts.geom.GeometryFactory) Logger(org.slf4j.Logger) Tags(com.apple.test.Tags) Throwables(com.google.common.base.Throwables) Matchers(org.hamcrest.Matchers) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) FileInputStream(java.io.FileInputStream) InputStreamReader(java.io.InputStreamReader) Index(com.apple.foundationdb.record.metadata.Index) Expressions.value(com.apple.foundationdb.record.metadata.Key.Expressions.value) FDBException(com.apple.foundationdb.FDBException) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) BufferedReader(java.io.BufferedReader) Collections(java.util.Collections) GeometryFactory(org.locationtech.jts.geom.GeometryFactory) Message(com.google.protobuf.Message) FDBException(com.apple.foundationdb.FDBException) Point(org.locationtech.jts.geom.Point) TopologyException(org.locationtech.jts.geom.TopologyException) Geometry(org.locationtech.jts.geom.Geometry) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) GeoJsonReader(org.locationtech.jts.io.geojson.GeoJsonReader) Coordinate(org.locationtech.jts.geom.Coordinate) CompletionException(java.util.concurrent.CompletionException) ParseException(org.locationtech.jts.io.ParseException) Pair(org.apache.commons.lang3.tuple.Pair) Test(org.junit.jupiter.api.Test) Tag(org.junit.jupiter.api.Tag)

Aggregations

FDBIndexedRecord (com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord)5 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)5 Message (com.google.protobuf.Message)5 Test (org.junit.jupiter.api.Test)5 BunchedMapScanEntry (com.apple.foundationdb.map.BunchedMapScanEntry)2 IndexEntry (com.apple.foundationdb.record.IndexEntry)2 KeyValueLogMessage (com.apple.foundationdb.record.logging.KeyValueLogMessage)2 Index (com.apple.foundationdb.record.metadata.Index)2 PlanMatchers.hasTupleString (com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString)2 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)2 Tuple (com.apple.foundationdb.tuple.Tuple)2 ArrayList (java.util.ArrayList)2 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)2 FDBException (com.apple.foundationdb.FDBException)1 BunchedMap (com.apple.foundationdb.map.BunchedMap)1 Bindings (com.apple.foundationdb.record.Bindings)1 EvaluationContext (com.apple.foundationdb.record.EvaluationContext)1 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)1 RecordCursor (com.apple.foundationdb.record.RecordCursor)1 RecordCursorIterator (com.apple.foundationdb.record.RecordCursorIterator)1