Examples with FDBRecordStore com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore used on opensource projects

Example 26 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method clique.

@Test
public void clique() throws Exception {
    final JoinedRecordTypeBuilder clique = metaDataBuilder.addJoinedRecordType("Clique");
    clique.addConstituent("type_a", "TypeA");
    clique.addConstituent("type_b", "TypeB");
    clique.addConstituent("type_c", "TypeC");
    clique.addJoin("type_a", "type_b_rec_no", "type_b", "rec_no");
    clique.addJoin("type_b", "type_c_rec_no", "type_c", "rec_no");
    clique.addJoin("type_c", "type_a_rec_no", "type_a", "rec_no");
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        for (int i = 0; i < 3; i++) {
            TestRecordsJoinIndexProto.TypeA.Builder typeA = TestRecordsJoinIndexProto.TypeA.newBuilder();
            typeA.setRecNo(100 + i).setTypeBRecNo(200 + i);
            recordStore.saveRecord(typeA.build());
            TestRecordsJoinIndexProto.TypeB.Builder typeB = TestRecordsJoinIndexProto.TypeB.newBuilder();
            typeB.setRecNo(200 + i).setTypeCRecNo(300 + i);
            recordStore.saveRecord(typeB.build());
            TestRecordsJoinIndexProto.TypeC.Builder typeC = TestRecordsJoinIndexProto.TypeC.newBuilder();
            typeC.setRecNo(300 + i).setTypeARecNo(100 + i);
            recordStore.saveRecord(typeC.build());
        }
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        final SyntheticRecordPlanner planner = new SyntheticRecordPlanner(recordStore);
        SyntheticRecordPlan plan1 = planner.scanForType(recordStore.getRecordMetaData().getSyntheticRecordType("Clique"));
        Multiset<Tuple> expected1 = ImmutableMultiset.of(Tuple.from(-1, Tuple.from(100), Tuple.from(200), Tuple.from(300)), Tuple.from(-1, Tuple.from(101), Tuple.from(201), Tuple.from(301)), Tuple.from(-1, Tuple.from(102), Tuple.from(202), Tuple.from(302)));
        Multiset<Tuple> results1 = HashMultiset.create(plan1.execute(recordStore).map(FDBSyntheticRecord::getPrimaryKey).asList().join());
        assertEquals(expected1, results1);
        // Make sure that the extra join condition is checked.
        TestRecordsJoinIndexProto.TypeC.Builder typeC = TestRecordsJoinIndexProto.TypeC.newBuilder();
        typeC.setRecNo(301).setTypeARecNo(999);
        recordStore.saveRecord(typeC.build());
        SyntheticRecordPlan plan2 = planner.scanForType(recordStore.getRecordMetaData().getSyntheticRecordType("Clique"));
        Multiset<Tuple> expected2 = ImmutableMultiset.of(Tuple.from(-1, Tuple.from(100), Tuple.from(200), Tuple.from(300)), Tuple.from(-1, Tuple.from(102), Tuple.from(202), Tuple.from(302)));
        Multiset<Tuple> results2 = HashMultiset.create(plan2.execute(recordStore).map(FDBSyntheticRecord::getPrimaryKey).asList().join());
        assertEquals(expected2, results2);
    }
}

Example 27 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method multiFieldKeys.

@Test
public void multiFieldKeys() throws Exception {
    metaDataBuilder.getRecordType("MySimpleRecord").setPrimaryKey(concatenateFields("num_value", "rec_no"));
    metaDataBuilder.getRecordType("MyOtherRecord").setPrimaryKey(concatenateFields("num_value", "rec_no"));
    final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("MultiFieldJoin");
    joined.addConstituent("simple", "MySimpleRecord");
    joined.addConstituent("other", "MyOtherRecord");
    // TODO: Not supported alternative would be to join concatenateFields("num_value", "other_rec_no") with concatenateFields("num_value", "rec_no").
    joined.addJoin("simple", "num_value", "other", "num_value");
    joined.addJoin("simple", "other_rec_no", "other", "rec_no");
    metaDataBuilder.addIndex(joined, new Index("simple.str_value_other.num_value_3", concat(field("simple").nest("str_value"), field("other").nest("num_value_3"))));
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        for (int n = 1; n <= 2; n++) {
            for (int i = 0; i < 3; i++) {
                for (int j = 0; j < i; j++) {
                    TestRecordsJoinIndexProto.MySimpleRecord.Builder simple = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder();
                    simple.setNumValue(n);
                    simple.setRecNo(100 * i + j).setOtherRecNo(1000 + i);
                    simple.setStrValue((i + j) % 2 == 0 ? "even" : "odd");
                    recordStore.saveRecord(simple.build());
                }
                TestRecordsJoinIndexProto.MyOtherRecord.Builder other = TestRecordsJoinIndexProto.MyOtherRecord.newBuilder();
                other.setNumValue(n);
                other.setRecNo(1000 + i);
                other.setNumValue3(i);
                recordStore.saveRecord(other.build());
            }
        }
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        List<FDBSyntheticRecord> recs = recordStore.scanIndex(recordStore.getRecordMetaData().getIndex("simple.str_value_other.num_value_3"), IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from("even", 2)), null, ScanProperties.FORWARD_SCAN).mapPipelined(entry -> recordStore.loadSyntheticRecord(entry.getPrimaryKey()), 1).asList().join();
        for (FDBSyntheticRecord record : recs) {
            TestRecordsJoinIndexProto.MySimpleRecord.Builder simple = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder();
            TestRecordsJoinIndexProto.MyOtherRecord.Builder other = TestRecordsJoinIndexProto.MyOtherRecord.newBuilder();
            simple.mergeFrom(record.getConstituent("simple").getRecord());
            other.mergeFrom(record.getConstituent("other").getRecord());
            assertEquals(200, simple.getRecNo());
            assertEquals(1002, other.getRecNo());
            assertEquals(record.getPrimaryKey(), record.getRecordType().getPrimaryKey().evaluateSingleton(record).toTuple());
        }
    }
}

Example 28 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method nestedRepeated.

@Test
public void nestedRepeated() throws Exception {
    final KeyExpression key = field("repeated", KeyExpression.FanType.FanOut).nest("nums", KeyExpression.FanType.FanOut);
    metaDataBuilder.addIndex("NestedA", "repeatedA", key);
    metaDataBuilder.addIndex("NestedB", "repeatedB", key);
    final JoinedRecordTypeBuilder nested = metaDataBuilder.addJoinedRecordType("NestedRepeated");
    nested.addConstituent("nested_a", "NestedA");
    nested.addConstituent("nested_b", "NestedB");
    nested.addJoin("nested_a", key, "nested_b", key);
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        TestRecordsJoinIndexProto.NestedA.Builder nestedA = TestRecordsJoinIndexProto.NestedA.newBuilder();
        nestedA.setRecNo(101);
        nestedA.addRepeatedBuilder().addNums(1).addNums(2);
        nestedA.addRepeatedBuilder().addNums(3).addNums(4);
        recordStore.saveRecord(nestedA.build());
        nestedA.setRecNo(102);
        nestedA.clearRepeated();
        nestedA.addRepeatedBuilder().addNums(2);
        recordStore.saveRecord(nestedA.build());
        TestRecordsJoinIndexProto.NestedB.Builder nestedB = TestRecordsJoinIndexProto.NestedB.newBuilder();
        nestedB.setRecNo(201);
        nestedB.addRepeatedBuilder().addNums(2).addNums(4);
        recordStore.saveRecord(nestedB.build());
        nestedB.setRecNo(202);
        nestedB.clearRepeated();
        nestedB.addRepeatedBuilder().addNums(1).addNums(3);
        nestedB.addRepeatedBuilder().addNums(2);
        recordStore.saveRecord(nestedB.build());
        nestedB.setRecNo(203);
        nestedB.clearRepeated();
        recordStore.saveRecord(nestedB.build());
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        final SyntheticRecordPlanner planner = new SyntheticRecordPlanner(recordStore);
        SyntheticRecordPlan plan1 = planner.scanForType(recordStore.getRecordMetaData().getSyntheticRecordType("NestedRepeated"));
        Multiset<Tuple> expected1 = ImmutableMultiset.of(Tuple.from(-1, Tuple.from(101), Tuple.from(201)), Tuple.from(-1, Tuple.from(101), Tuple.from(202)), Tuple.from(-1, Tuple.from(102), Tuple.from(201)), Tuple.from(-1, Tuple.from(102), Tuple.from(202)));
        Multiset<Tuple> results1 = HashMultiset.create(plan1.execute(recordStore).map(FDBSyntheticRecord::getPrimaryKey).asList().join());
        assertEquals(expected1, results1);
        FDBStoredRecord<Message> record = recordStore.loadRecord(Tuple.from(101));
        SyntheticRecordFromStoredRecordPlan plan2 = planner.fromStoredType(record.getRecordType(), false);
        // TODO: IN can generate duplicates from repeated field (https://github.com/FoundationDB/fdb-record-layer/issues/98)
        Multiset<Tuple> expected2 = ImmutableMultiset.of(Tuple.from(-1, Tuple.from(101), Tuple.from(201)), Tuple.from(-1, Tuple.from(101), Tuple.from(201)), Tuple.from(-1, Tuple.from(101), Tuple.from(202)), Tuple.from(-1, Tuple.from(101), Tuple.from(202)), Tuple.from(-1, Tuple.from(101), Tuple.from(202)));
        Multiset<Tuple> results2 = HashMultiset.create(plan2.execute(recordStore, record).map(FDBSyntheticRecord::getPrimaryKey).asList().join());
        assertEquals(expected2, results2);
    }
}

Example 29 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project SBK by kmgowda.

the class FdbRecord method openStorage.

@Override
public void openStorage(final ParameterOptions params) throws IOException {
    db = FDBDatabaseFactory.instance().getDatabase(config.cFile);
    // Define the keyspace for our application
    KeySpace keySpace = new KeySpace(new KeySpaceDirectory(config.keySpace, KeySpaceDirectory.KeyType.STRING, config.keySpace));
    // Get the path where our record store will be rooted
    KeySpacePath path = keySpace.path(config.keySpace);
    RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(FdbRecordLayerProto.getDescriptor());
    metaDataBuilder.getRecordType("Record").setPrimaryKey(Key.Expressions.field("recordID"));
    recordStoreProvider = context -> FDBRecordStore.newBuilder().setMetaDataProvider(metaDataBuilder).setContext(context).setKeySpacePath(path).createOrOpen();
    if (params.getWritersCount() > 0) {
        db.run(context -> {
            FDBRecordStore recordStore = recordStoreProvider.apply(context);
            recordStore.deleteAllRecords();
            return null;
        });
    }
}

Example 30 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project SBK by kmgowda.

the class FdbRecordMultiReader method recordRead.

@Override
public void recordRead(DataType<ByteString> dType, int size, Time time, Status status, PerlChannel perlChannel) throws EOFException, IOException {
    final int recs = params.getRecordsPerSync();
    status.startTime = time.getCurrentTime();
    final Status ret = db.run(context -> {
        long startKey = key;
        Status stat = new Status();
        FDBRecordStore recordStore = recordStoreProvider.apply(context);
        FDBStoredRecord<Message> storedRecord;
        for (int i = 0; i < recs; i++) {
            storedRecord = recordStore.loadRecord(Tuple.from(startKey++));
            if (storedRecord != null) {
                FdbRecordLayerProto.Record record = FdbRecordLayerProto.Record.newBuilder().mergeFrom(storedRecord.getRecord()).build();
                stat.bytes += record.getData().size();
                stat.records += 1;
            }
        }
        return stat;
    });
    if (ret.records == 0) {
        throw new EOFException();
    }
    status.records = ret.records;
    status.bytes = ret.bytes;
    status.endTime = time.getCurrentTime();
    key += recs;
    cnt += recs;
    perlChannel.send(status.startTime, status.endTime, status.bytes, status.records);
}