Examples with Key com.apple.foundationdb.record.metadata.Key used on opensource projects

Example 26 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class FunctionKeyRecordTest method testCoveringIndexFunction.

@Test
public void testCoveringIndexFunction() throws Exception {
    // This index parses each entry of str_array_field of the form "X:Y:Z" and produces a keyWithValue
    // index of the form X -> (Y, Z).
    final Index index = new Index("covering", keyWithValue(function("regex", concat(field("str_array_field", KeyExpression.FanType.FanOut), value("(\\d+):(\\w+):(\\d+)"), value("LONG"), value("STRING"), value("LONG"))), 1));
    final RecordMetaDataHook hook = metadata -> {
        RecordTypeBuilder type = metadata.getRecordType("StringRecordId");
        type.setPrimaryKey(field("rec_id"));
        metadata.addIndex(type, index);
    };
    final BiFunction<Integer, Integer, String> makeId = (id1, id2) -> id1 + ":" + Character.toString((char) ('a' + id2)) + ":" + id2;
    // Create some records
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        for (int i = 0; i < 5; i++) {
            TestRecords8Proto.StringRecordId.Builder builder = TestRecords8Proto.StringRecordId.newBuilder().setRecId("record_" + i).setIntValue(i);
            for (int j = 0; j < 4; j++) {
                builder.addStrArrayField(makeId.apply((i * 4) + j, j));
            }
            recordStore.saveRecord(builder.build());
        }
        commit(context);
    }
    Function<FDBIndexedRecord<Message>, TestRecords8Proto.StringRecordId> validate = message -> {
        final Tuple key = message.getIndexEntry().getKey();
        final Tuple value = message.getIndexEntry().getValue();
        // Key is X,<record_id> where X is the first part of the str_array_field
        Assertions.assertEquals(2, key.size());
        // Value is the last two pieces of the str_array_field
        Assertions.assertEquals(2, value.size());
        // Check the record itself
        final TestRecords8Proto.StringRecordId record = TestRecords8Proto.StringRecordId.newBuilder().mergeFrom(message.getRecord()).build();
        // Get the portions of the key and the value that are needed to reconstruct the
        // full value that is supposed to be stored in the str_array_field.
        final int id1 = (int) key.getLong(0);
        final int id2 = (int) value.getLong(1);
        final int recordId = (id1 / 4);
        Assertions.assertEquals(recordId, record.getIntValue());
        Assertions.assertEquals("record_" + recordId, record.getRecId());
        Assertions.assertTrue(record.getStrArrayFieldList().contains(makeId.apply(id1, id2)), "str_array_field does not contain entry");
        Assertions.assertEquals(Character.toString((char) ('a' + id2)), value.getString(0));
        return record;
    };
    // let's just scan all of the records
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        List<FDBIndexedRecord<Message>> messages = recordStore.getRecordContext().asyncToSync(FDBStoreTimer.Waits.WAIT_SCAN_INDEX_RECORDS, recordStore.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList());
        Assertions.assertEquals(20, messages.size(), "Wrong record count");
        for (FDBIndexedRecord<Message> message : messages) {
            validate.apply(message);
        }
    }
    // Next, scan a subset of them based upon the first value of the covering index
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, hook);
        TupleRange range = new TupleRange(Tuple.from(2), Tuple.from(4), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
        List<FDBIndexedRecord<Message>> messages = recordStore.getRecordContext().asyncToSync(FDBStoreTimer.Waits.WAIT_SCAN_INDEX_RECORDS, recordStore.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).asList());
        Assertions.assertEquals(2, messages.size(), "Wrong record count");
        for (FDBIndexedRecord<Message> message : messages) {
            TestRecords8Proto.StringRecordId record = validate.apply(message);
            Assertions.assertTrue(record.getIntValue() == 0, "Invalid int value");
        }
    }
}

Example 27 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class KeyComparisonsTest method keyComparator.

@Test
@Tag(Tags.Slow)
public void keyComparator() {
    Random r = new Random(0x5ca1ab1e);
    // Type 1: Integers.
    List<Key.Evaluated> intKeys = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        intKeys.add(Key.Evaluated.scalar(r.nextInt()));
    }
    intKeys.add(Key.Evaluated.NULL);
    testKeyComparator(intKeys);
    // Type 2: Int-Strings
    List<Key.Evaluated> intStringKeys = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        intStringKeys.add(Key.Evaluated.concatenate(r.nextInt(5), randomString(r)));
    }
    intStringKeys.add(Key.Evaluated.NULL);
    testKeyComparator(intStringKeys);
    // Type 3: Int-String lists
    List<Key.Evaluated> intStringListKeys = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        int length = (int) (Math.abs(r.nextGaussian() + 0.2) * 10);
        int first = r.nextInt(5);
        List<String> list = new ArrayList<>();
        for (int j = 0; j < length; j++) {
            list.add(randomString(r));
            if (r.nextDouble() < 0.4) {
                intStringListKeys.add(Key.Evaluated.concatenate(first, new ArrayList<>(list)));
            }
        }
        intStringListKeys.add(Key.Evaluated.concatenate(first, list));
    }
    intStringListKeys.add(Key.Evaluated.NULL);
    testKeyComparator(intStringListKeys);
    // Type 4: Int-Byte arrays
    List<Key.Evaluated> intByteArraysKeys = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        int length = (int) (Math.abs(r.nextGaussian() + 0.2) * 10);
        byte[] arr = new byte[length];
        r.nextBytes(arr);
        intByteArraysKeys.add(Key.Evaluated.concatenate(r.nextInt(5), arr));
    }
    intByteArraysKeys.add(Key.Evaluated.NULL);
    testKeyComparator(intByteArraysKeys);
    // Type 5: Mix of ints and int byte arrays.
    List<Key.Evaluated> mixedKeys = new ArrayList<>();
    mixedKeys.addAll(intKeys);
    mixedKeys.addAll(intByteArraysKeys);
    for (Key.Evaluated key : intByteArraysKeys) {
        List<Object> list = key.toList();
        if (list.size() > 0) {
            mixedKeys.add(Key.Evaluated.scalar(list.get(0)));
        }
    }
    testKeyComparator(mixedKeys);
}

Example 28 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class FDBCoveringIndexQueryTest method coveringMultiValue.

/**
 * Verify that an index can be covering if some of the required fields are in the value part of the index.
 */
@DualPlannerTest
void coveringMultiValue() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$num_value_unique");
        metaData.addIndex("MySimpleRecord", new Index("multi_index_value", field("num_value_unique"), field("num_value_2"), IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_unique").greaterThan(990)).setSort(field("num_value_unique")).setRequiredResults(Arrays.asList(field("num_value_unique"), field("num_value_2"))).build();
    // Covering(Index(multi_index_value ([990],>) -> [num_value_2: VALUE[0], num_value_unique: KEY[0], rec_no: KEY[1]])
    RecordQueryPlan plan = planner.plan(query);
    final BindingMatcher<? extends RecordQueryPlan> planMatcher = coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index_value")).and(scanComparisons(range("([990],>")))));
    assertMatchesExactly(plan, planMatcher);
    assertEquals(-782505942, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(450250048, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(368845640, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertTrue(myrec.getNumValueUnique() > 990);
                assertEquals(myrec.getNumValue2(), (999 - i) % 3);
                i++;
            }
        }
        assertEquals(10, i);
        assertDiscardedNone(context);
    }
}

Example 29 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class FDBCoveringIndexQueryTest method coveringWithAdditionalFilter.

/**
 * Verify that a filter not satisfied by the index scan itself but using fields present in the index
 * can still allow a covering scan with the filter on the partial records.
 */
@DualPlannerTest
void coveringWithAdditionalFilter() throws Exception {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$num_value_3_indexed");
        metaData.addIndex("MySimpleRecord", new Index("multi_index", "num_value_3_indexed", "num_value_2"));
    };
    complexQuerySetup(hook);
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_3_indexed").lessThan(1), Query.field("num_value_2").lessThan(2))).setRequiredResults(Collections.singletonList(field("num_value_3_indexed"))).build();
    // Covering(Index(multi_index ([null],[1])) -> [num_value_2: KEY[1], num_value_3_indexed: KEY[0], rec_no: KEY[2]]) | num_value_2 LESS_THAN 2
    RecordQueryPlan plan = planner.plan(query);
    if (planner instanceof RecordQueryPlanner) {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = filterPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index")).and(scanComparisons(range("([null],[1])")))))).where(queryComponents(exactly(equalsObject(Query.field("num_value_2").lessThan(2)))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(-1374002128, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(1359983418, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-1492450855, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        final BindingMatcher<? extends RecordQueryPlan> planMatcher = predicatesFilterPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi_index")).and(scanComparisons(range("([null],[1])")))))).where(predicates(only(valuePredicate(fieldValue("num_value_2"), new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 2)))));
        assertMatchesExactly(plan, planMatcher);
        assertEquals(762957369, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
        assertEquals(-2135370744, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
        assertEquals(-692837721, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    }
}

Example 30 with Key

use of com.apple.foundationdb.record.metadata.Key in project fdb-record-layer by FoundationDB.

the class FDBCoveringIndexQueryTest method queryCoveringAggregate.

/**
 * Verify that selecting the group key and the aggregate function from a grouped aggregate index can be planned
 * by a covering aggregate index.
 */
@Test
void queryCoveringAggregate() {
    Index sumIndex = new Index("value3sum", field("num_value_3_indexed").groupBy(Key.Expressions.concatenateFields("str_value_indexed", "num_value_2")), IndexTypes.SUM);
    RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", sumIndex);
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        for (int i = 0; i < 20; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed((i & 1) == 1 ? "odd" : "even");
            recBuilder.setNumValue2(i % 3);
            recBuilder.setNumValue3Indexed(i % 5);
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("even")).setRequiredResults(Arrays.asList(field("str_value_indexed"), field("num_value_2"))).build();
    // This is here since the main planner doesn't currently support planning aggregates, so it's basically a
    // separate "mini-planner".
    // TODO: Support aggregate planning in the main query planner (https://github.com/FoundationDB/fdb-record-layer/issues/14)
    RecordQueryPlan plan = ((RecordQueryPlanner) planner).planCoveringAggregateIndex(query, "value3sum");
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        int i = 0;
        try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(Objects.requireNonNull(plan)).asIterator()) {
            while (cursor.hasNext()) {
                FDBQueriedRecord<Message> rec = cursor.next();
                TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
                myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
                assertEquals("even", myrec.getStrValueIndexed());
                int sum = 0;
                for (int j = 0; j < 20; j += 2) {
                    if (j % 3 == myrec.getNumValue2()) {
                        sum += j % 5;
                    }
                }
                assertEquals(sum, Objects.requireNonNull(rec.getIndexEntry()).getValue().getLong(0));
                i++;
            }
        }
        assertEquals(3, i);
        assertDiscardedNone(context);
    }
}