Search in sources :

Example 21 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class FunctionKeyIndexTest method testIndexScan.

@Test
public void testIndexScan() throws Exception {
    // Note: the substr() function is defined by KeyExpressionTest.java
    Index index = new Index("substr_index", function("substr", concat(field("str_value"), value(0), value(3))));
    Records records = Records.create();
    for (int i = 0; i < 10; i++) {
        records.add(i, "ab" + Character.toString((char) ('c' + i)) + "_" + i);
    }
    saveRecords(records, index);
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, index);
        TupleRange range = new TupleRange(Tuple.from("abd"), Tuple.from("abg_5"), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_INCLUSIVE);
        List<FDBIndexedRecord<Message>> results = recordStore.getRecordContext().asyncToSync(FDBStoreTimer.Waits.WAIT_SCAN_INDEX_RECORDS, recordStore.scanIndexRecords(index.getName(), IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).asList());
        assertEquals(4, results.size());
        for (FDBIndexedRecord<Message> indexedRecord : results) {
            TypesRecord record = fromMessage(indexedRecord.getRecord());
            assertTrue(records.contains(record), "Record does not exist");
            assertEquals(record.getStrValue().substring(0, 3), indexedRecord.getIndexEntry().getKey().getString(0));
        }
    }
}
Also used : Message(com.google.protobuf.Message) Index(com.apple.foundationdb.record.metadata.Index) TupleRange(com.apple.foundationdb.record.TupleRange) TypesRecord(com.apple.foundationdb.record.TestDataTypesProto.TypesRecord) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 22 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method joinIndex.

@Test
public void joinIndex() throws Exception {
    metaDataBuilder.addIndex("MySimpleRecord", "other_rec_no");
    final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("Simple_Other");
    joined.addConstituent("simple", "MySimpleRecord");
    joined.addConstituent("other", "MyOtherRecord");
    joined.addJoin("simple", "other_rec_no", "other", "rec_no");
    metaDataBuilder.addIndex(joined, new Index("simple.str_value_other.num_value_3", concat(field("simple").nest("str_value"), field("other").nest("num_value_3"))));
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < i; j++) {
                TestRecordsJoinIndexProto.MySimpleRecord.Builder simple = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder();
                simple.setRecNo(100 * i + j).setOtherRecNo(1000 + i);
                simple.setStrValue((i + j) % 2 == 0 ? "even" : "odd");
                recordStore.saveRecord(simple.build());
            }
            TestRecordsJoinIndexProto.MyOtherRecord.Builder other = TestRecordsJoinIndexProto.MyOtherRecord.newBuilder();
            other.setRecNo(1000 + i);
            other.setNumValue3(i);
            recordStore.saveRecord(other.build());
        }
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        final Index index = recordStore.getRecordMetaData().getIndex("simple.str_value_other.num_value_3");
        final TupleRange range = new ScanComparisons.Builder().addEqualityComparison(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, "even")).addInequalityComparison(new Comparisons.SimpleComparison(Comparisons.Type.GREATER_THAN, 1)).build().toTupleRange();
        List<Tuple> expected1 = Arrays.asList(Tuple.from("even", 2, -1, Tuple.from(200), Tuple.from(1002)));
        List<Tuple> results1 = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).map(IndexEntry::getKey).asList().join();
        assertEquals(expected1, results1);
        FDBStoredRecord<Message> record = recordStore.loadRecord(Tuple.from(201));
        TestRecordsJoinIndexProto.MySimpleRecord.Builder recordBuilder = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder().mergeFrom(record.getRecord());
        recordBuilder.setStrValue("even");
        recordStore.saveRecord(recordBuilder.build());
        List<Tuple> expected2 = Arrays.asList(Tuple.from("even", 2, -1, Tuple.from(200), Tuple.from(1002)), Tuple.from("even", 2, -1, Tuple.from(201), Tuple.from(1002)));
        List<Tuple> results2 = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).map(IndexEntry::getKey).asList().join();
        assertEquals(expected2, results2);
        recordStore.deleteRecord(Tuple.from(1002));
        List<Tuple> expected3 = Arrays.asList();
        List<Tuple> results3 = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).map(IndexEntry::getKey).asList().join();
        assertEquals(expected3, results3);
    }
}
Also used : Message(com.google.protobuf.Message) JoinedRecordTypeBuilder(com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) IndexEntry(com.apple.foundationdb.record.IndexEntry) Index(com.apple.foundationdb.record.metadata.Index) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) JoinedRecordTypeBuilder(com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Test(org.junit.jupiter.api.Test)

Example 23 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method buildJoinIndex.

@Test
public void buildJoinIndex() throws Exception {
    metaDataBuilder.addIndex("MySimpleRecord", "other_rec_no");
    final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("Simple_Other");
    joined.addConstituent("simple", "MySimpleRecord");
    joined.addConstituent("other", "MyOtherRecord");
    joined.addJoin("simple", "other_rec_no", "other", "rec_no");
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        for (int i = 0; i < 3; i++) {
            TestRecordsJoinIndexProto.MySimpleRecord.Builder simple = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder();
            simple.setRecNo(i).setOtherRecNo(1000 + i);
            simple.setNumValue2(i * 2);
            recordStore.saveRecord(simple.build());
            TestRecordsJoinIndexProto.MyOtherRecord.Builder other = TestRecordsJoinIndexProto.MyOtherRecord.newBuilder();
            other.setRecNo(1000 + i);
            other.setNumValue3(i * 3);
            recordStore.saveRecord(other.build());
        }
        context.commit();
    }
    metaDataBuilder.addIndex(joined, new Index("simple.num_value_2_other.num_value_3", concat(field("simple").nest("num_value_2"), field("other").nest("num_value_3"))));
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        final Index index = recordStore.getRecordMetaData().getIndex("simple.num_value_2_other.num_value_3");
        final TupleRange range = new ScanComparisons.Builder().addEqualityComparison(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 2)).addEqualityComparison(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 3)).build().toTupleRange();
        List<Tuple> expected1 = Arrays.asList(Tuple.from(2, 3, -1, Tuple.from(1), Tuple.from(1001)));
        List<Tuple> results1 = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).map(IndexEntry::getKey).asList().join();
        assertEquals(expected1, results1);
    }
}
Also used : JoinedRecordTypeBuilder(com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) IndexEntry(com.apple.foundationdb.record.IndexEntry) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) Index(com.apple.foundationdb.record.metadata.Index) JoinedRecordTypeBuilder(com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) TupleRange(com.apple.foundationdb.record.TupleRange) Tuple(com.apple.foundationdb.tuple.Tuple) Test(org.junit.jupiter.api.Test)

Example 24 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreTest method testOverlappingPrimaryKey.

@Test
public void testOverlappingPrimaryKey() {
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestRecordsWithHeaderProto.getDescriptor());
        builder.getRecordType("MyRecord").setPrimaryKey(field("header").nest(concatenateFields("path", "rec_no")));
        builder.addIndex("MyRecord", "MyRecord$path_str", concat(field("header").nest("path"), field("str_value")));
        RecordMetaData metaData = builder.getRecordMetaData();
        createOrOpenRecordStore(context, metaData);
        TestRecordsWithHeaderProto.MyRecord.Builder recBuilder = TestRecordsWithHeaderProto.MyRecord.newBuilder();
        TestRecordsWithHeaderProto.HeaderRecord.Builder headerBuilder = recBuilder.getHeaderBuilder();
        headerBuilder.setPath("aaa");
        headerBuilder.setRecNo(1);
        recBuilder.setStrValue("hello");
        recordStore.saveRecord(recBuilder.build());
        headerBuilder.setPath("aaa");
        headerBuilder.setRecNo(2);
        recBuilder.setStrValue("goodbye");
        recordStore.saveRecord(recBuilder.build());
        headerBuilder.setPath("zzz");
        headerBuilder.setRecNo(3);
        recBuilder.setStrValue("end");
        recordStore.saveRecord(recBuilder.build());
        List<List<Object>> rows = new ArrayList<>();
        Index index = metaData.getIndex("MyRecord$path_str");
        ScanComparisons comparisons = ScanComparisons.from(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, "aaa"));
        TupleRange range = comparisons.toTupleRange();
        try (RecordCursor<IndexEntry> cursor = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN)) {
            cursor.forEach(row -> rows.add(row.getKey().getItems())).join();
        }
        assertEquals(Arrays.asList(Arrays.asList("aaa", "goodbye", 2L), Arrays.asList("aaa", "hello", 1L)), rows);
    }
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) DescriptorProtos(com.google.protobuf.DescriptorProtos) DELETE_INDEX_ENTRY(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer.Events.DELETE_INDEX_ENTRY) Arrays(java.util.Arrays) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) ByteBuffer(java.nio.ByteBuffer) RecordSerializer(com.apple.foundationdb.record.provider.common.RecordSerializer) RecordSerializationException(com.apple.foundationdb.record.provider.common.RecordSerializationException) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) RecordMetaDataProto(com.apple.foundationdb.record.RecordMetaDataProto) RecordMetaDataOptionsProto(com.apple.foundationdb.record.RecordMetaDataOptionsProto) Is.is(org.hamcrest.core.Is.is) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) Tag(org.junit.jupiter.api.Tag) MethodSource(org.junit.jupiter.params.provider.MethodSource) TestRecords7Proto(com.apple.foundationdb.record.TestRecords7Proto) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) Collectors(java.util.stream.Collectors) TupleRange(com.apple.foundationdb.record.TupleRange) ByteString(com.google.protobuf.ByteString) Test(org.junit.jupiter.api.Test) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) TestRecordsWithHeaderProto(com.apple.foundationdb.record.TestRecordsWithHeaderProto) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.containsString(org.hamcrest.Matchers.containsString) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) TestNoUnionProto(com.apple.foundationdb.record.TestNoUnionProto) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) TestRecordsDuplicateUnionFields(com.apple.foundationdb.record.TestRecordsDuplicateUnionFields) DynamicMessage(com.google.protobuf.DynamicMessage) Descriptors(com.google.protobuf.Descriptors) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) CompletableFuture(java.util.concurrent.CompletableFuture) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Strings(com.google.common.base.Strings) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) Matchers.lessThan(org.hamcrest.Matchers.lessThan) Assumptions.assumeTrue(org.junit.jupiter.api.Assumptions.assumeTrue) Matchers.hasSize(org.hamcrest.Matchers.hasSize) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) ExecutionException(java.util.concurrent.ExecutionException) Consumer(java.util.function.Consumer) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) TestRecordsImportProto(com.apple.foundationdb.record.TestRecordsImportProto) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) Collections(java.util.Collections) SAVE_INDEX_ENTRY(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer.Events.SAVE_INDEX_ENTRY) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) ArrayList(java.util.ArrayList) IndexEntry(com.apple.foundationdb.record.IndexEntry) Index(com.apple.foundationdb.record.metadata.Index) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) ScanComparisons(com.apple.foundationdb.record.query.plan.ScanComparisons) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) List(java.util.List) ArrayList(java.util.ArrayList) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 25 with TupleRange

use of com.apple.foundationdb.record.TupleRange in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testNoBoundaryPrimaryKeysImpl.

public void testNoBoundaryPrimaryKeysImpl() {
    final FDBDatabaseFactory factory = FDBDatabaseFactory.instance();
    factory.setLocalityProvider(MockedLocalityUtil.instance());
    FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
    final String indexName = "MySimpleRecord$num_value_unique";
    try (FDBRecordContext context = database.openContext()) {
        openSimpleRecordStore(context, TEST_SPLIT_HOOK);
        recordStore.markIndexWriteOnly(indexName).join();
        commit(context);
    }
    String bigOlString = Strings.repeat("x", SplitHelper.SPLIT_RECORD_SIZE + 2);
    saveAndSplitSimpleRecord(1, bigOlString, 1);
    saveAndSplitSimpleRecord(2, bigOlString, 2);
    OnlineIndexer indexer;
    List<Tuple> boundaryPrimaryKeys;
    TupleRange range;
    try (FDBRecordContext context = database.openContext()) {
        openSimpleRecordStore(context, TEST_SPLIT_HOOK);
        Index index = recordStore.getRecordMetaData().getIndex(indexName);
        // The indexer only uses recordStore as a prototype so does not require the original record store is still
        // active.
        indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setRecordStore(recordStore).setIndex(index).build();
        range = recordStore.context.asyncToSync(FDBStoreTimer.Waits.WAIT_BUILD_ENDPOINTS, indexer.buildEndpoints());
        logger.info("The endpoints are " + range);
        MockedLocalityUtil.init(new ArrayList<>(Arrays.asList(recordStore.recordsSubspace().pack(1), recordStore.recordsSubspace().pack(2))), 0);
        boundaryPrimaryKeys = recordStore.context.asyncToSync(FDBStoreTimer.Waits.WAIT_GET_BOUNDARY, recordStore.getPrimaryKeyBoundaries(range.getLow(), range.getHigh()).asList());
        assertEquals(0, boundaryPrimaryKeys.size());
        logger.info("The boundary primary keys are " + boundaryPrimaryKeys);
        commit(context);
    }
    // Test splitIndexBuildRange.
    assertEquals(1, indexer.splitIndexBuildRange(Integer.MAX_VALUE, Integer.MAX_VALUE).size());
    indexer.close();
    database.close();
}
Also used : TupleRange(com.apple.foundationdb.record.TupleRange) Index(com.apple.foundationdb.record.metadata.Index) Tuple(com.apple.foundationdb.tuple.Tuple)

Aggregations

TupleRange (com.apple.foundationdb.record.TupleRange)48 Tuple (com.apple.foundationdb.tuple.Tuple)29 Nonnull (javax.annotation.Nonnull)28 Message (com.google.protobuf.Message)21 Index (com.apple.foundationdb.record.metadata.Index)16 Test (org.junit.jupiter.api.Test)16 ScanProperties (com.apple.foundationdb.record.ScanProperties)15 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)14 Nullable (javax.annotation.Nullable)14 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)11 IndexEntry (com.apple.foundationdb.record.IndexEntry)11 List (java.util.List)11 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)10 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)10 RecordCursor (com.apple.foundationdb.record.RecordCursor)10 IndexScanType (com.apple.foundationdb.record.IndexScanType)9 CompletableFuture (java.util.concurrent.CompletableFuture)9 Range (com.apple.foundationdb.Range)8 API (com.apple.foundationdb.annotation.API)8 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)8