Examples with IndexRecordFunction com.apple.foundationdb.record.metadata.IndexRecordFunction used on opensource projects

Example 1 with IndexRecordFunction

use of com.apple.foundationdb.record.metadata.IndexRecordFunction in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method rankJoinIndex.

@Test
public void rankJoinIndex() throws Exception {
    final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("JoinedForRank");
    joined.addConstituent("simple", "MySimpleRecord");
    joined.addConstituent("other", "MyOtherRecord");
    joined.addJoin("simple", "other_rec_no", "other", "rec_no");
    final GroupingKeyExpression group = field("simple").nest("num_value_2").groupBy(field("other").nest("num_value"));
    metaDataBuilder.addIndex(joined, new Index("simple.num_value_2_by_other.num_value", group, IndexTypes.RANK));
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        for (int i = 0; i < 4; i++) {
            for (int j = 0; j < i; j++) {
                TestRecordsJoinIndexProto.MySimpleRecord.Builder simple = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder();
                simple.setRecNo(100 * i + j).setOtherRecNo(1000 + i);
                simple.setNumValue2(i + j);
                recordStore.saveRecord(simple.build());
            }
            TestRecordsJoinIndexProto.MyOtherRecord.Builder other = TestRecordsJoinIndexProto.MyOtherRecord.newBuilder();
            other.setRecNo(1000 + i);
            other.setNumValue(i % 2);
            recordStore.saveRecord(other.build());
        }
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        Index index = recordStore.getRecordMetaData().getIndex("simple.num_value_2_by_other.num_value");
        RecordCursor<IndexEntry> cursor = recordStore.scanIndex(index, IndexScanType.BY_RANK, TupleRange.allOf(Tuple.from(0, 1)), null, ScanProperties.FORWARD_SCAN);
        Tuple pkey = cursor.first().get().map(IndexEntry::getPrimaryKey).orElse(null);
        assertFalse(cursor.getNext().hasNext());
        // 201, 1002 and 200, 1003 both have score 3, but in different groups.
        assertEquals(Tuple.from(-1, Tuple.from(201), Tuple.from(1002)), pkey);
        FDBSyntheticRecord record = recordStore.loadSyntheticRecord(pkey).join();
        IndexRecordFunction<Long> rankFunction = ((IndexRecordFunction<Long>) Query.rank(group).getFunction()).cloneWithIndex(index.getName());
        assertEquals(1, recordStore.evaluateRecordFunction(rankFunction, record).join().longValue());
    }
}

Example 2 with IndexRecordFunction

use of com.apple.foundationdb.record.metadata.IndexRecordFunction in project fdb-record-layer by FoundationDB.

the class OnlineIndexerBuildRankIndexTest method rankRebuild.

private void rankRebuild(@Nonnull List<TestRecords1Proto.MySimpleRecord> records, @Nullable List<TestRecords1Proto.MySimpleRecord> recordsWhileBuilding, int agents, boolean overlap) {
    final Index index = new Index("newRankIndex", field("num_value_2").ungrouped(), IndexTypes.RANK);
    final IndexRecordFunction<Long> recordFunction = (IndexRecordFunction<Long>) Query.rank("num_value_2").getFunction();
    final Runnable beforeBuild = new Runnable() {

        @SuppressWarnings("try")
        @Override
        public void run() {
            try (FDBRecordContext context = openContext()) {
                for (TestRecords1Proto.MySimpleRecord record : records) {
                    try {
                        recordStore.evaluateRecordFunction(recordFunction, createStoredMessage(record)).join();
                        fail("Somehow evaluated rank");
                    } catch (RecordCoreException e) {
                        assertEquals("Record function rank(Field { 'num_value_2' None} group 1) requires appropriate index on MySimpleRecord", e.getMessage());
                    }
                }
                // Either we get an exception, or the record store is empty.
                try {
                    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.rank("num_value_2").equalsValue(0L)).build();
                    RecordQueryPlan plan = planner.plan(query);
                    assertEquals("Scan(<,>) | [MySimpleRecord] | rank(Field { 'num_value_2' None} group 1) EQUALS 0", plan.toString());
                    Optional<?> first = recordStore.executeQuery(plan).first().join();
                    assertTrue(!first.isPresent(), "non-empty range with rank rebuild");
                } catch (CompletionException e) {
                    assertNotNull(e.getCause());
                    assertThat(e.getCause(), instanceOf(RecordCoreException.class));
                    assertEquals("Record function rank(Field { 'num_value_2' None} group 1) requires appropriate index on MySimpleRecord", e.getCause().getMessage());
                }
            }
        }
    };
    List<TestRecords1Proto.MySimpleRecord> updatedRecords;
    if (recordsWhileBuilding == null || recordsWhileBuilding.size() == 0) {
        updatedRecords = records;
    } else {
        updatedRecords = updated(records, recordsWhileBuilding);
    }
    final Runnable afterBuild = new Runnable() {

        @SuppressWarnings("try")
        @Override
        public void run() {
            try (FDBRecordContext context = openContext()) {
                for (TestRecords1Proto.MySimpleRecord record : updatedRecords) {
                    try {
                        recordStore.evaluateRecordFunction(recordFunction, createStoredMessage(record)).join();
                        fail("Somehow evaluated rank");
                    } catch (RecordCoreException e) {
                        assertEquals("Record function rank(Field { 'num_value_2' None} group 1) requires appropriate index on MySimpleRecord", e.getMessage());
                    }
                }
                // Either we get an exception, or the record store is empty.
                try {
                    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.rank("num_value_2").equalsValue(0L)).build();
                    RecordQueryPlan plan = planner.plan(query);
                    assertEquals("Scan(<,>) | [MySimpleRecord] | rank(Field { 'num_value_2' None} group 1) EQUALS 0", plan.toString());
                    Optional<?> first = recordStore.executeQuery(plan).first().join();
                    assertTrue(!first.isPresent(), "non-empty range with rank rebuild");
                } catch (CompletionException e) {
                    assertNotNull(e.getCause());
                    assertThat(e.getCause(), instanceOf(RecordCoreException.class));
                    assertEquals("Record function rank(Field { 'num_value_2' None} group 1) requires appropriate index on MySimpleRecord", e.getCause().getMessage());
                }
            }
        }
    };
    TreeSet<Integer> values = new TreeSet<>();
    boolean hasNull = false;
    for (TestRecords1Proto.MySimpleRecord record : updatedRecords) {
        if (!record.hasNumValue2()) {
            hasNull = true;
        } else {
            values.add(record.getNumValue2());
        }
    }
    long curr = 0;
    Map<Integer, Long> ranks = new HashMap<>();
    if (hasNull) {
        curr += 1;
    }
    for (Integer value : values) {
        ranks.put(value, curr);
        curr += 1;
    }
    final Runnable afterReadable = new Runnable() {

        @SuppressWarnings("try")
        @Override
        public void run() {
            try (FDBRecordContext context = openContext()) {
                for (TestRecords1Proto.MySimpleRecord record : updatedRecords) {
                    Long rank = recordStore.evaluateRecordFunction(recordFunction, createStoredMessage(record)).join();
                    if (!record.hasNumValue2()) {
                        assertEquals(0L, rank.longValue());
                    } else {
                        assertEquals(ranks.get(record.getNumValue2()), rank);
                    }
                    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.rank("num_value_2").equalsValue(rank)).build();
                    RecordQueryPlan plan = planner.plan(query);
                    assertEquals("Index(newRankIndex [[" + rank + "],[" + rank + "]] BY_RANK)", plan.toString());
                    Optional<TestRecords1Proto.MySimpleRecord> retrieved = recordStore.executeQuery(plan).map(rec -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(rec.getRecord()).build()).filter(rec -> rec.getRecNo() == record.getRecNo()).first().join();
                    assertTrue(retrieved.isPresent(), "empty range after rank index build");
                    assertEquals(record, retrieved.get());
                }
            }
        }
    };
    singleRebuild(records, recordsWhileBuilding, agents, overlap, false, index, beforeBuild, afterBuild, afterReadable);
}

Example 3 with IndexRecordFunction

use of com.apple.foundationdb.record.metadata.IndexRecordFunction in project fdb-record-layer by FoundationDB.

the class RankIndexTest method writeOnlyLookup.

@Test
public void writeOnlyLookup() throws Exception {
    assertThrows(RecordCoreException.class, () -> {
        fdb = FDBDatabaseFactory.instance().getDatabase();
        try (FDBRecordContext context = openContext()) {
            openRecordStore(context);
            recordStore.markIndexWriteOnly("rank_by_gender").join();
            // Re-open to reload state.
            openRecordStore(context);
            TestRecordsRankProto.BasicRankedRecord record = TestRecordsRankProto.BasicRankedRecord.newBuilder().setName("achilles").setGender("M").setScore(10).build();
            FDBStoredRecord<Message> storedRecord = recordStore.saveRecord(record);
            IndexRecordFunction<Long> function = (IndexRecordFunction<Long>) Query.rank(Key.Expressions.field("score").groupBy(Key.Expressions.field("gender"))).getFunction();
            recordStore.evaluateRecordFunction(function, storedRecord);
        }
    });
}

Example 4 with IndexRecordFunction

use of com.apple.foundationdb.record.metadata.IndexRecordFunction in project fdb-record-layer by FoundationDB.

the class RankComparisons method findComparison.

private void findComparison(@Nonnull QueryRecordFunctionWithComparison comparison, @Nonnull List<Index> indexes, @Nonnull List<QueryComponent> potentialGroupFilters, @Nonnull AtomicInteger counter) {
    RecordFunction<?> recordFunction = comparison.getFunction();
    // TODO: Should share with indexMaintainerForAggregateFunction
    // TODO: Move index-specific query planning behavior outside of planner (https://github.com/FoundationDB/fdb-record-layer/issues/17)
    List<String> requiredIndexTypes;
    if (recordFunction.getName().equals(FunctionNames.RANK)) {
        requiredIndexTypes = Arrays.asList(IndexTypes.RANK, IndexTypes.TIME_WINDOW_LEADERBOARD);
    } else if (recordFunction.getName().equals(FunctionNames.TIME_WINDOW_RANK)) {
        requiredIndexTypes = Collections.singletonList(IndexTypes.TIME_WINDOW_LEADERBOARD);
    } else {
        requiredIndexTypes = null;
    }
    if (requiredIndexTypes != null) {
        final GroupingKeyExpression operand = ((IndexRecordFunction) recordFunction).getOperand();
        Optional<Index> matchingIndex = indexes.stream().filter(index -> requiredIndexTypes.contains(index.getType()) && index.getRootExpression().equals(operand)).min(Comparator.comparing(Index::getColumnSize));
        if (matchingIndex.isPresent()) {
            final KeyExpression groupBy = operand.getGroupingSubKey();
            final List<QueryComponent> groupFilters = new ArrayList<>();
            final List<Comparisons.Comparison> groupComparisons = new ArrayList<>();
            if (!GroupingValidator.findGroupKeyFilters(potentialGroupFilters, groupBy, groupFilters, groupComparisons)) {
                return;
            }
            QueryComponent substitute = null;
            String bindingName = null;
            final Comparisons.Type comparisonType = comparison.getComparison().getType();
            if (!operand.createsDuplicates() && !comparisonType.isUnary()) {
                bindingName = Bindings.Internal.RANK.bindingName(Integer.toString(counter.getAndIncrement()));
                Comparisons.Comparison substituteComparison = new Comparisons.ParameterComparison(comparisonType, bindingName, Bindings.Internal.RANK);
                final KeyExpression grouped = operand.getGroupedSubKey();
                if (grouped instanceof FieldKeyExpression) {
                    substitute = new FieldWithComparison(((FieldKeyExpression) grouped).getFieldName(), substituteComparison);
                } else if (grouped instanceof NestingKeyExpression) {
                    NestingKeyExpression nesting = (NestingKeyExpression) grouped;
                    if (nesting.getChild() instanceof FieldKeyExpression) {
                        substitute = new NestedField(nesting.getParent().getFieldName(), new FieldWithComparison(((FieldKeyExpression) nesting.getChild()).getFieldName(), substituteComparison));
                    }
                }
                if (substitute == null) {
                    bindingName = null;
                }
            }
            comparisons.put(comparison, new RankComparison(comparison, matchingIndex.get(), groupFilters, groupComparisons, substitute, bindingName));
        }
    }
}