Examples with FDBRecord com.apple.foundationdb.record.provider.foundationdb.FDBRecord used on opensource projects

Example 1 with FDBRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecord in project fdb-record-layer by FoundationDB.

the class StandardIndexMaintainer method evaluateIndex.

/**
 * Apply the key and value expressions to a <code>record</code>.
 * @param <M> the message type of the record
 * @param record the record from which the index will extract its key and value
 * @return a list of index keys and values
 */
@Override
public <M extends Message> List<IndexEntry> evaluateIndex(@Nonnull FDBRecord<M> record) {
    final KeyExpression rootExpression = state.index.getRootExpression();
    final List<Key.Evaluated> indexKeys = rootExpression.evaluate(record);
    // so we have to tease them apart.
    if (rootExpression instanceof KeyWithValueExpression) {
        final KeyWithValueExpression keyWithValueExpression = (KeyWithValueExpression) rootExpression;
        return indexKeys.stream().map(key -> new IndexEntry(state.index, keyWithValueExpression.getKey(key), keyWithValueExpression.getValue(key))).collect(Collectors.toList());
    }
    return indexKeys.stream().map(key -> new IndexEntry(state.index, key)).collect(Collectors.toList());
}

Example 2 with FDBRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecord in project fdb-record-layer by FoundationDB.

the class TimeWindowLeaderboardIndexMaintainer method timeWindowRankAndEntry.

@Nonnull
public <M extends Message> CompletableFuture<Pair<Long, Tuple>> timeWindowRankAndEntry(@Nonnull FDBRecord<M> record, int type, long timestamp) {
    final List<IndexEntry> indexEntries = evaluateIndex(record);
    final CompletableFuture<TimeWindowLeaderboard> leaderboardFuture = oldestLeaderboardMatching(type, timestamp);
    return leaderboardFuture.thenCompose(leaderboard -> {
        if (leaderboard == null) {
            return CompletableFuture.completedFuture(null);
        }
        return groupOrderedScoreIndexKeys(indexEntries, leaderboard.getDirectory(), true).thenCompose(groupedScores -> {
            if (groupedScores.isEmpty()) {
                return CompletableFuture.completedFuture(null);
            }
            if (groupedScores.size() > 1) {
                throw new RecordCoreException("Record has more than one group of scores");
            }
            Map.Entry<Tuple, Collection<OrderedScoreIndexKey>> groupEntry = groupedScores.entrySet().iterator().next();
            Optional<OrderedScoreIndexKey> bestContainedScore = groupEntry.getValue().stream().filter(score -> leaderboard.containsTimestamp(score.timestamp)).findFirst();
            if (!bestContainedScore.isPresent()) {
                return CompletableFuture.completedFuture(null);
            }
            // bestContainedScore should be the one stored in the leaderboard's ranked set; get its rank there.
            final Tuple groupKey = groupEntry.getKey();
            return isHighScoreFirst(leaderboard.getDirectory(), groupKey).thenCompose(highScoreFirst -> {
                final OrderedScoreIndexKey indexKey = bestContainedScore.get();
                final Tuple leaderboardGroupKey = leaderboard.getSubspaceKey().addAll(groupKey);
                final Subspace extraSubspace = getSecondarySubspace();
                final Subspace rankSubspace = extraSubspace.subspace(leaderboardGroupKey);
                final RankedSet.Config leaderboardConfig = config.toBuilder().setNLevels(leaderboard.getNLevels()).build();
                final RankedSet rankedSet = new RankedSetIndexHelper.InstrumentedRankedSet(state, rankSubspace, leaderboardConfig);
                // Undo any negation needed to find entry.
                final Tuple entry = highScoreFirst ? negateScoreForHighScoreFirst(indexKey.scoreKey, 0) : indexKey.scoreKey;
                return RankedSetIndexHelper.rankForScore(state, rankedSet, indexKey.scoreKey, true).thenApply(rank -> Pair.of(rank, entry));
            });
        });
    });
}

Example 3 with FDBRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecord in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method queryOnVersion.

@ParameterizedTest(name = "queryOnVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnVersion(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
    List<TestRecords1Proto.MyOtherRecord> otherRecords = IntStream.range(0, 30).mapToObj(id -> TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(id * 2 + 1).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
    Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
    Iterator<TestRecords1Proto.MyOtherRecord> otherIterator = otherRecords.iterator();
    while (simpleIterator.hasNext()) {
        try (FDBRecordContext context = openContext(simpleVersionHook)) {
            int done = 0;
            while (simpleIterator.hasNext() && done != 5) {
                recordStore.saveRecord(simpleIterator.next());
                recordStore.saveRecord(otherIterator.next());
                done += 1;
            }
            context.commit();
        }
    }
    // Query all records.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = Stream.concat(simpleRecords.stream().map(MySimpleRecord::getRecNo), otherRecords.stream().map(TestRecords1Proto.MyOtherRecord::getRecNo)).sorted().collect(Collectors.toList());
        FDBRecordVersion last = null;
        List<Long> receivedKeys = new ArrayList<>();
        int totalSeen = 0;
        while (true) {
            RecordQueryPlan plan;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
                plan = planner.plan(query);
                assertEquals("Index(globalVersion <,>)", plan.toString());
            } else {
                RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().greaterThan(last)).setSort(VersionKeyExpression.VERSION).build();
                plan = planner.plan(query);
                assertEquals("Index(globalVersion ([" + last.toVersionstamp() + "],>)", plan.toString());
            }
            RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator();
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBQueriedRecord<Message> record = cursor.next();
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals(simpleRecords.size() + otherRecords.size(), totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query MySimpleRecord based on value.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
        List<Long> receivedKeys = new ArrayList<>();
        FDBRecordVersion last = null;
        int totalSeen = 0;
        while (true) {
            RecordCursorIterator<? extends FDBRecord<Message>> cursor;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version [[0],[0]])", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
            } else {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]])", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
            }
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBRecord<Message> record = cursor.next();
                MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
                assertEquals(0, simpleRecord.getNumValue2());
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(simpleRecord.getRecNo());
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals((simpleRecords.size() + 1) / 2, totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query that requires also filtering
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0 && rec.getNumValue3Indexed() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
        List<Long> receivedKeys = new ArrayList<>();
        FDBRecordVersion last = null;
        int totalSeen = 0;
        while (true) {
            RecordCursorIterator<? extends FDBRecord<Message>> cursor;
            if (last == null) {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version [[0],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
            } else {
                RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
                RecordQueryPlan plan = planner.plan(query);
                assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
                cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
            }
            boolean hasAny = false;
            while (cursor.hasNext()) {
                hasAny = true;
                FDBRecord<Message> record = cursor.next();
                MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
                assertEquals(0, simpleRecord.getNumValue2());
                assertTrue(record.hasVersion());
                if (last != null) {
                    assertThat(last, lessThan(record.getVersion()));
                }
                last = record.getVersion();
                receivedKeys.add(simpleRecord.getRecNo());
                totalSeen += 1;
            }
            if (!hasAny) {
                break;
            }
        }
        assertEquals(simpleRecords.size() / 6, totalSeen);
        assertEquals(expectedKeys, receivedKeys);
    }
    // Query that can't be satisfied with an index scan.
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        // Preliminary query to get a read version.
        RecordQuery prelimQuery = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
        RecordQueryPlan prelimPlan = planner.plan(prelimQuery);
        FDBRecordVersion chosenVersion = recordStore.executeQuery(prelimPlan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asList().thenApply(list -> list.get(list.size() - 1).getVersion()).join();
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.version().greaterThan(chosenVersion)).setSort(field("num_value_3_indexed")).build();
        RecordQueryPlan plan = planner.plan(query);
        assertEquals("Index(MySimpleRecord$num_value_3_indexed <,>) | version GREATER_THAN " + chosenVersion.toString(), plan.toString());
        List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
        int last = -1;
        for (FDBQueriedRecord<Message> record : records) {
            MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
            assertThat(last, lessThanOrEqualTo(simpleRecord.getNumValue3Indexed()));
            assertTrue(record.hasVersion());
            assertThat(chosenVersion, lessThan(record.getVersion()));
            last = simpleRecord.getNumValue3Indexed();
        }
        assertEquals(simpleRecords.size() - 5, records.size());
    }
}

Example 4 with FDBRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecord in project fdb-record-layer by FoundationDB.

the class UnionIntersectionTest method unevenInterleavedIndexScan.

@ValueSource(ints = { 1, 2, 3, 4, 5 })
@ParameterizedTest(name = "unevenInterleavedIndexScan() [{0}]")
public void unevenInterleavedIndexScan(int innerLimit) throws Exception {
    final ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(innerLimit).build());
    final ScanComparisons zeros = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 0)), Collections.emptySet());
    final ScanComparisons others = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 1)), Collections.emptySet());
    verifyUnionWithInnerLimits(cont -> recordStore.scanIndexRecords("MySimpleRecord$num_value_3_indexed", IndexScanType.BY_VALUE, zeros.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), cont -> recordStore.scanIndexRecords("MySimpleRecord$num_value_3_indexed", IndexScanType.BY_VALUE, others.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), LongStream.range(0L, 100L).iterator());
}

Example 5 with FDBRecord

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecord in project fdb-record-layer by FoundationDB.

the class UnionIntersectionTest method interleavedIndexScan.

@ValueSource(ints = { 1, 2, 3, 4, 5 })
@ParameterizedTest(name = "interleavedIndexScan() [{0}]")
public void interleavedIndexScan(int innerLimit) throws Exception {
    final ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(innerLimit).build());
    final ScanComparisons evenComparisons = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, "even")), Collections.emptySet());
    final ScanComparisons oddComparisons = new ScanComparisons(Collections.singletonList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, "odd")), Collections.emptySet());
    verifyUnionWithInnerLimits(cont -> recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, evenComparisons.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), cont -> recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, oddComparisons.toTupleRange(), cont, scanProperties).map(rec -> (FDBRecord<Message>) rec), LongStream.range(0L, 100L).iterator());
}