Examples with TupleRange com.apple.foundationdb.record.TupleRange used on opensource projects

Example 26 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testBoundaryPrimaryKeysImpl.

public void testBoundaryPrimaryKeysImpl() {
    final FDBDatabaseFactory factory = FDBDatabaseFactory.instance();
    factory.setLocalityProvider(MockedLocalityUtil.instance());
    FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    final String indexName = "MySimpleRecord$num_value_unique";
    try (FDBRecordContext context = database.openContext()) {
        openSimpleRecordStore(context, TEST_SPLIT_HOOK);
        recordStore.markIndexWriteOnly(indexName).join();
        commit(context);
    }
    ArrayList<byte[]> keys = new ArrayList<>();
    String bigOlString = Strings.repeat("x", SplitHelper.SPLIT_RECORD_SIZE + 2);
    for (int i = 0; i < 50; i++) {
        saveAndSplitSimpleRecord(i, bigOlString, i);
        keys.add(recordStore.recordsSubspace().pack(i));
    }
    OnlineIndexer indexer;
    List<Tuple> boundaryPrimaryKeys;
    TupleRange range;
    try (FDBRecordContext context = database.openContext()) {
        openSimpleRecordStore(context, TEST_SPLIT_HOOK);
        // 3 <= rangeCount <= size
        MockedLocalityUtil.init(keys, new Random().nextInt(keys.size() - 2) + 3);
        Index index = recordStore.getRecordMetaData().getIndex(indexName);
        // The indexer only uses recordStore as a prototype so does not require the original record store is still
        // active.
        indexer = OnlineIndexer.newBuilder().setDatabase(database).setRecordStore(recordStore).setIndex(index).build();
        range = recordStore.context.asyncToSync(FDBStoreTimer.Waits.WAIT_BUILD_ENDPOINTS, indexer.buildEndpoints());
        logger.info("The endpoints are " + range);
        boundaryPrimaryKeys = recordStore.context.asyncToSync(FDBStoreTimer.Waits.WAIT_GET_BOUNDARY, recordStore.getPrimaryKeyBoundaries(range.getLow(), range.getHigh()).asList());
        logger.info("The boundary primary keys are " + boundaryPrimaryKeys);
        commit(context);
    }
    int boundaryPrimaryKeysSize = boundaryPrimaryKeys.size();
    assertTrue(boundaryPrimaryKeysSize > 2, "the test is meaningless if the records are not across boundaries");
    assertThat(boundaryPrimaryKeys.get(0), greaterThanOrEqualTo(Tuple.from(-25L * 39)));
    assertThat(boundaryPrimaryKeys.get(boundaryPrimaryKeysSize - 1), lessThanOrEqualTo(Tuple.from(24L * 39)));
    assertEquals(boundaryPrimaryKeys.stream().sorted().distinct().collect(Collectors.toList()), boundaryPrimaryKeys, "the list should be sorted without duplication.");
    for (Tuple boundaryPrimaryKey : boundaryPrimaryKeys) {
        assertEquals(1, boundaryPrimaryKey.size(), "primary keys should be a single value");
    }
    // Test splitIndexBuildRange.
    assertEquals(1, indexer.splitIndexBuildRange(Integer.MAX_VALUE, Integer.MAX_VALUE).size(), "the range is not split when it cannot be split to at least minSplit ranges");
    // to test splitting into fewer than the default number of split points
    checkSplitIndexBuildRange(1, 2, null, indexer);
    // to test splitting into fewer than the default number of split points
    checkSplitIndexBuildRange(boundaryPrimaryKeysSize / 2, boundaryPrimaryKeysSize - 2, null, indexer);
    checkSplitIndexBuildRange(boundaryPrimaryKeysSize / 2, boundaryPrimaryKeysSize, null, indexer);
    List<Pair<Tuple, Tuple>> oneRangePerSplit = getOneRangePerSplit(range, boundaryPrimaryKeys);
    // to test exactly one range for each split
    checkSplitIndexBuildRange(boundaryPrimaryKeysSize / 2, boundaryPrimaryKeysSize + 1, oneRangePerSplit, indexer);
    checkSplitIndexBuildRange(boundaryPrimaryKeysSize / 2, boundaryPrimaryKeysSize + 2, oneRangePerSplit, indexer);
    // to test that integer overflow isn't a problem
    checkSplitIndexBuildRange(boundaryPrimaryKeysSize / 2, Integer.MAX_VALUE, oneRangePerSplit, indexer);
    indexer.close();
    database.close();
}

Example 27 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class FunctionKeyRecordTest method testCoveringIndexFunction.

@Test
public void testCoveringIndexFunction() throws Exception {
    // This index parses each entry of str_array_field of the form "X:Y:Z" and produces a keyWithValue
    // index of the form X -> (Y, Z).
    final Index index = new Index("covering", keyWithValue(function("regex", concat(field("str_array_field", KeyExpression.FanType.FanOut), value("(\\d+):(\\w+):(\\d+)"), value("LONG"), value("STRING"), value("LONG"))), 1));
    final RecordMetaDataHook hook = metadata -> {
        RecordTypeBuilder type = metadata.getRecordType("StringRecordId");
        type.setPrimaryKey(field("rec_id"));
        metadata.addIndex(type, index);
    };
    final BiFunction<Integer, Integer, String> makeId = (id1, id2) -> id1 + ":" + Character.toString((char) ('a' + id2)) + ":" + id2;
    // Create some records
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        for (int i = 0; i < 5; i++) {
            TestRecords8Proto.StringRecordId.Builder builder = TestRecords8Proto.StringRecordId.newBuilder().setRecId("record_" + i).setIntValue(i);
            for (int j = 0; j < 4; j++) {
                builder.addStrArrayField(makeId.apply((i * 4) + j, j));
            }
            recordStore.saveRecord(builder.build());
        }
        commit(context);
    }
    Function<FDBIndexedRecord<Message>, TestRecords8Proto.StringRecordId> validate = message -> {
        final Tuple key = message.getIndexEntry().getKey();
        final Tuple value = message.getIndexEntry().getValue();
        // Key is X,<record_id> where X is the first part of the str_array_field
        Assertions.assertEquals(2, key.size());
        // Value is the last two pieces of the str_array_field
        Assertions.assertEquals(2, value.size());
        // Check the record itself
        final TestRecords8Proto.StringRecordId record = TestRecords8Proto.StringRecordId.newBuilder().mergeFrom(message.getRecord()).build();
        // Get the portions of the key and the value that are needed to reconstruct the
        // full value that is supposed to be stored in the str_array_field.
        final int id1 = (int) key.getLong(0);
        final int id2 = (int) value.getLong(1);
        final int recordId = (id1 / 4);
        Assertions.assertEquals(recordId, record.getIntValue());
        Assertions.assertEquals("record_" + recordId, record.getRecId());
        Assertions.assertTrue(record.getStrArrayFieldList().contains(makeId.apply(id1, id2)), "str_array_field does not contain entry");
        Assertions.assertEquals(Character.toString((char) ('a' + id2)), value.getString(0));
        return record;
    };
    // let's just scan all of the records
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        List<FDBIndexedRecord<Message>> messages = recordStore.getRecordContext().asyncToSync(FDBStoreTimer.Waits.WAIT_SCAN_INDEX_RECORDS, recordStore.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList());
        Assertions.assertEquals(20, messages.size(), "Wrong record count");
        for (FDBIndexedRecord<Message> message : messages) {
            validate.apply(message);
        }
    }
    // Next, scan a subset of them based upon the first value of the covering index
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        TupleRange range = new TupleRange(Tuple.from(2), Tuple.from(4), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
        List<FDBIndexedRecord<Message>> messages = recordStore.getRecordContext().asyncToSync(FDBStoreTimer.Waits.WAIT_SCAN_INDEX_RECORDS, recordStore.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).asList());
        Assertions.assertEquals(2, messages.size(), "Wrong record count");
        for (FDBIndexedRecord<Message> message : messages) {
            TestRecords8Proto.StringRecordId record = validate.apply(message);
            Assertions.assertTrue(record.getIntValue() == 0, "Invalid int value");
        }
    }
}

Example 28 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class RankIndexTest method checkRankScan.

@Test
public void checkRankScan() throws Exception {
    TupleRange range = new TupleRange(Tuple.from(0L), Tuple.from(2L), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        int i = 0;
        try (RecordCursorIterator<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("BasicRankedRecord$score", IndexScanType.BY_RANK, range, null, ScanProperties.FORWARD_SCAN).asIterator()) {
            while (cursor.hasNext()) {
                FDBIndexedRecord<Message> rec = cursor.next();
                TestRecordsRankProto.BasicRankedRecord.Builder myrec = TestRecordsRankProto.BasicRankedRecord.newBuilder();
                myrec.mergeFrom(rec.getRecord());
                assertTrue(myrec.getScore() < 200);
                i++;
            }
        }
        assertEquals(2, i);
    }
    range = TupleRange.allOf(Tuple.from("M", 0L));
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        try (RecordCursor<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("rank_by_gender", IndexScanType.BY_RANK, range, null, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(1).build()))) {
            FDBIndexedRecord<Message> rec = cursor.getNext().get();
            TestRecordsRankProto.BasicRankedRecord.Builder myrec = TestRecordsRankProto.BasicRankedRecord.newBuilder();
            myrec.mergeFrom(rec.getRecord());
            assertEquals("hector", myrec.getName());
            assertEquals(75, myrec.getScore());
        }
    }
}

Example 29 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class LeaderboardIndexTest method rebuildChangeDirection.

@Test
public void rebuildChangeDirection() {
    Leaderboards leaderboards = new GroupedNestedLeaderboards();
    basicSetup(leaderboards, false);
    try (FDBRecordContext context = openContext()) {
        leaderboards.openRecordStore(context, false);
        TupleRange game_1 = TupleRange.allOf(Tuple.from("game-1"));
        assertEquals(Arrays.asList("achilles", "hector", "hecuba", "patroclus"), leaderboards.scanIndexByRank(game_1).map(leaderboards::getName).asList().join());
        TimeWindowScanRange top_2_ten_units = new TimeWindowScanRange(TEN_UNITS, 10100, new TupleRange(Tuple.from("game-1", 0), Tuple.from("game-1", 1), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE));
        assertEquals(Arrays.asList("achilles", "hector"), leaderboards.scanIndexByTimeWindow(top_2_ten_units).map(leaderboards::getName).asList().join());
    }
    try (FDBRecordContext context = openContext()) {
        leaderboards.openRecordStore(context, false);
        metrics.reset();
        TimeWindowLeaderboardWindowUpdateResult result = leaderboards.updateWindows(true, 10100);
        assertTrue(result.isChanged());
        assertTrue(result.isRebuilt());
        assertEquals(1, metrics.getCount(FDBStoreTimer.Events.REBUILD_INDEX));
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        leaderboards.openRecordStore(context, false);
        TupleRange game_1 = TupleRange.allOf(Tuple.from("game-1"));
        assertEquals(Arrays.asList("patroclus", "hecuba", "achilles", "hector"), leaderboards.scanIndexByRank(game_1).map(leaderboards::getName).asList().join());
        TimeWindowScanRange top_2_ten_units = new TimeWindowScanRange(TEN_UNITS, 10100, new TupleRange(Tuple.from("game-1", 0), Tuple.from("game-1", 1), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE));
        assertEquals(Arrays.asList("achilles", "hector"), leaderboards.scanIndexByTimeWindow(top_2_ten_units).map(leaderboards::getName).asList().join());
    }
}

Example 30 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class LeaderboardIndexTest method deleteWhere.

@Test
public void deleteWhere() {
    Leaderboards leaderboards = new GroupedNestedLeaderboards();
    leaderboards.buildMetaData(metaDataBuilder -> {
        // Make the primary keys include the game so we can range delete game's records.
        metaDataBuilder.getRecordType("NestedLeaderboardRecord").setPrimaryKey(Key.Expressions.concat(Key.Expressions.field("game_id"), Key.Expressions.field("name")));
        metaDataBuilder.getRecordType("FlatLeaderboardRecord").setPrimaryKey(Key.Expressions.concat(Key.Expressions.field("game_id"), Key.Expressions.field("name")));
        leaderboards.addIndex(metaDataBuilder);
    });
    try (FDBRecordContext context = openContext()) {
        leaderboards.openRecordStore(context, true);
        leaderboards.updateWindows(true, 10100);
        addInitialScores(leaderboards);
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        leaderboards.openRecordStore(context, false);
        TupleRange game_1 = TupleRange.allOf(Tuple.from("game-1"));
        TupleRange game_2 = TupleRange.allOf(Tuple.from("game-2"));
        assertEquals(Arrays.asList("patroclus", "hecuba", "achilles", "hector"), leaderboards.scanIndexByRank(game_1).map(leaderboards::getName).asList().join());
        assertEquals(Arrays.asList("helen"), leaderboards.scanIndexByRank(game_2).map(leaderboards::getName).asList().join());
        leaderboards.recordStore.deleteRecordsWhere(Query.field("game_id").equalsValue("game-1"));
        assertEquals(Collections.emptyList(), leaderboards.scanIndexByRank(game_1).map(leaderboards::getName).asList().join());
        assertEquals(Arrays.asList("helen"), leaderboards.scanIndexByRank(game_2).map(leaderboards::getName).asList().join());
    }
}