Examples with Index com.apple.foundationdb.record.metadata.Index used on opensource projects

Example 91 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method andOrQueryWithDuplicate.

@Test
void andOrQueryWithDuplicate() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        saveRecords(100, 200);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        // Covering(Index(rec_no_by_str_num2 [[odd, 3],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 [[odd, 0],[odd, 0]] BY_GROUP) -> [rec_no: KEY[2]]) BITOR Covering(Index(rec_no_by_str_num2 [[odd, 3],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 [[odd, 4],[odd, 4]] BY_GROUP) -> [rec_no: KEY[2]])
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.or(Query.and(Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(0)), Query.and(Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4)))));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1] BITOR [0] BITAND [2]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 3],[odd, 3]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 0],[odd, 0]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("[[odd, 4],[odd, 4]]"))))))));
        assertEquals(1788540340, queryPlan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1021904334, queryPlan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1583681802, queryPlan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> ((i % 7) == 3 && (i % 5) == 0) || ((i % 7) == 3 && (i % 5) == 4)).collect(Collectors.toList())));
    }
}

Example 92 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method andQueryPosition.

@Test
void andQueryPosition() {
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        saveRecords(100, 200);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
        setupPlanner(null);
        // Covering(Index(rec_no_by_str_num2 ([odd, 3, 150],[odd, 3]] BY_GROUP) -> [rec_no: KEY[2]]) BITAND Covering(Index(rec_no_by_str_num3 ([odd, 4, 150],[odd, 4]] BY_GROUP) -> [rec_no: KEY[2]])
        final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.field("num_value_2").equalsValue(3), Query.field("num_value_3").equalsValue(4), Query.field("rec_no").greaterThan(150)));
        assertThat(queryPlan, compositeBitmap(hasToString("[0] BITAND [1]"), Arrays.asList(coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num2"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("([odd, 3, 150],[odd, 3]]"))))), coveringIndexScan(indexScan(allOf(indexName("rec_no_by_str_num3"), indexScanType(IndexScanType.BY_GROUP), bounds(hasTupleString("([odd, 4, 150],[odd, 4]]"))))))));
        assertEquals(-1911273393, queryPlan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(2018486938, queryPlan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(1342370457, queryPlan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        assertThat(collectOnBits(queryPlan.execute(recordStore).map(FDBQueriedRecord::getIndexEntry)), equalTo(IntStream.range(151, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && (i % 5) == 4).collect(Collectors.toList())));
    }
}

Example 93 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class BitmapValueIndexTest method nonPrimaryKey.

@Test
void nonPrimaryKey() {
    final RecordMetaDataHook num_by_num3_hook = metadata -> {
        metadata.addIndex(metadata.getRecordType("MySimpleRecord"), new Index("num_by_num3", concatenateFields("num_value_3", "num_value_unique").group(1), IndexTypes.BITMAP_VALUE, SMALL_BITMAP_OPTIONS));
    };
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(num_by_num3_hook));
        saveRecords(0, 100);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        createOrOpenRecordStore(context, metaData(num_by_num3_hook));
        assertThat(collectOnBits(recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("num_by_num3"), IndexScanType.BY_GROUP, TupleRange.allOf(Tuple.from(2)), null, ScanProperties.FORWARD_SCAN)), equalTo(IntStream.range(1000, 1100).boxed().filter(i -> (i % 5) == 2).collect(Collectors.toList())));
    }
}

Example 94 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class UnionIntersectionTest method indexScansByPrimaryKey.

/**
 * Create cursors that correspond to union or intersection query and validate that using the custom comparison
 * key works.
 */
@Test
public void indexScansByPrimaryKey() throws Exception {
    final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final Index strValueIndex = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
        final Index numValue3Index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$num_value_3_indexed");
        List<Long> recNos = IntersectionCursor.create((IndexEntry entry) -> TupleHelpers.subTuple(entry.getKey(), 1, entry.getKey().size()).getItems(), false, (byte[] leftContinuation) -> recordStore.scanIndex(strValueIndex, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from("even")), leftContinuation, scanProperties), (byte[] rightContinuation) -> recordStore.scanIndex(numValue3Index, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from(1)), rightContinuation, scanProperties), null, recordStore.getTimer()).mapPipelined(indexEntry -> recordStore.loadRecordAsync(strValueIndex.getEntryPrimaryKey(indexEntry.getKey())), recordStore.getPipelineSize(PipelineOperation.INDEX_TO_RECORD)).map(this::storedRecordRecNo).asList().get();
        assertEquals(LongStream.range(0, 100).filter(i -> i % 2 == 0).filter(i -> i % 3 != 0).boxed().collect(Collectors.toList()), recNos);
        assertDiscardedAtMost(50, context);
        recNos = UnionCursor.create((IndexEntry entry) -> TupleHelpers.subTuple(entry.getKey(), 1, entry.getKey().size()).getItems(), false, (byte[] leftContinuation) -> recordStore.scanIndex(strValueIndex, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from("even")), leftContinuation, scanProperties), (byte[] rightContinuation) -> recordStore.scanIndex(numValue3Index, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from(1)), rightContinuation, scanProperties), null, recordStore.getTimer()).mapPipelined(indexEntry -> recordStore.loadRecordAsync(strValueIndex.getEntryPrimaryKey(indexEntry.getKey())), recordStore.getPipelineSize(PipelineOperation.INDEX_TO_RECORD)).map(this::storedRecordRecNo).asList().get();
        assertEquals(LongStream.range(0, 100).filter(i -> i % 2 == 0 || i % 3 != 0).boxed().collect(Collectors.toList()), recNos);
        assertDiscardedAtMost(83, context);
        commit(context);
    }
}

Example 95 with Index

use of com.apple.foundationdb.record.metadata.Index in project fdb-record-layer by FoundationDB.

the class GeophileQueryTest method testDistance.

@Test
@Tag(Tags.Slow)
public void testDistance() throws Exception {
    final RecordMetaDataHook hook = md -> {
        md.addIndex("City", CITY_LOCATION_COVERING_INDEX);
    };
    loadCities(hook, 0);
    final int centerId = 5391959;
    final double distance = 1;
    final int scanLimit = 5000;
    final RecordQueryPlan scanPlan = distanceFilterScan(distance);
    final Set<Integer> scanResults = new HashSet<>();
    byte[] continuation = null;
    do {
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context, hook);
            EvaluationContext joinContext = bindCenter(centerId);
            ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build();
            RecordCursor<FDBQueriedRecord<Message>> recordCursor = scanPlan.execute(recordStore, joinContext, continuation, executeProperties);
            recordCursor.forEach(city -> {
                TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
                LOGGER.debug(String.format("Scan found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
                scanResults.add(cityBuilder.getGeoNameId());
            }).join();
            continuation = recordCursor.getNext().getContinuation().toBytes();
            commit(context);
        }
    } while (continuation != null);
    final RecordQueryPlan indexPlan = distanceSpatialQuery(distance, false);
    final Set<Integer> indexResults = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<FDBQueriedRecord<Message>> recordCursor = indexPlan.execute(recordStore, bindCenter(centerId));
        recordCursor.forEach(city -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
            LOGGER.debug(String.format("Index found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
            indexResults.add(cityBuilder.getGeoNameId());
        }).join();
        int given = timer.getCount(FDBStoreTimer.Counts.QUERY_FILTER_GIVEN);
        int passed = timer.getCount(FDBStoreTimer.Counts.QUERY_FILTER_PASSED);
        int discarded = timer.getCount(FDBStoreTimer.Counts.QUERY_DISCARDED);
        assertThat("Should have passed more than discarded", passed, greaterThan(discarded));
        commit(context);
    }
    final RecordQueryPlan coveringPlan = distanceSpatialQuery(distance, true);
    final Set<Integer> coveringResults = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        RecordCursor<FDBQueriedRecord<Message>> recordCursor = indexPlan.execute(recordStore, bindCenter(centerId));
        recordCursor.forEach(city -> {
            TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(city.getRecord());
            LOGGER.debug(String.format("Covering found %s: %s", cityBuilder.getGeoNameId(), cityBuilder.getName()));
            coveringResults.add(cityBuilder.getGeoNameId());
        }).join();
        commit(context);
    }
    assertEquals(scanResults, indexResults);
    assertEquals(scanResults, coveringResults);
}