Examples with FDBRecordStore com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore used on opensource projects

Example 6 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class TextIndexTest method textIndexPerf1000ParallelInsert.

@Tag(Tags.Performance)
@Test
public void textIndexPerf1000ParallelInsert() throws Exception {
    // Create 1000 records
    Random r = new Random();
    List<SimpleDocument> records = getRandomRecords(r, 1000);
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        recordStore.asBuilder().create();
        commit(context);
    }
    final FDBRecordStore.Builder storeBuilder = recordStore.asBuilder();
    long startTime = System.nanoTime();
    int oldMaxAttempts = FDBDatabaseFactory.instance().getMaxAttempts();
    FDBDatabaseFactory.instance().setMaxAttempts(Integer.MAX_VALUE);
    try {
        CompletableFuture<?>[] workerFutures = new CompletableFuture<?>[10];
        int recordsPerWorker = records.size() / workerFutures.length;
        for (int i = 0; i < workerFutures.length; i++) {
            List<SimpleDocument> workerDocs = records.subList(i * recordsPerWorker, (i + 1) * recordsPerWorker);
            CompletableFuture<Void> workerFuture = new CompletableFuture<>();
            Thread workerThread = new Thread(() -> {
                try {
                    for (int j = 0; j < workerDocs.size(); j += 10) {
                        // Use retry loop to catch not_committed errors
                        List<SimpleDocument> batchDocuments = workerDocs.subList(j, j + 10);
                        fdb.run(context -> {
                            try {
                                FDBRecordStore store = storeBuilder.copyBuilder().setContext(context).open();
                                for (SimpleDocument document : batchDocuments) {
                                    store.saveRecord(document);
                                }
                                return null;
                            } catch (RecordCoreException e) {
                                throw e;
                            } catch (Exception e) {
                                throw new RecordCoreException(e);
                            }
                        });
                    }
                    workerFuture.complete(null);
                } catch (RuntimeException e) {
                    workerFuture.completeExceptionally(e);
                }
            });
            workerThread.setName("insert-worker-" + i);
            workerThread.start();
            workerFutures[i] = workerFuture;
        }
        CompletableFuture.allOf(workerFutures).get();
        long endTime = System.nanoTime();
        LOGGER.info("performed 1000 parallel insertions in {} seconds.", (endTime - startTime) * 1e-9);
        printUsage();
    } finally {
        FDBDatabaseFactory.instance().setMaxAttempts(oldMaxAttempts);
    }
}

Example 7 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method readVersionFromStoredRecordInTwoStores.

/**
 * Store a record in one store, then store a different record in a different store with the same primary key
 * and validate that the version read for the first record matches the version written and not the version
 * for the second record (which could happen if the local version cache leaks information between stores).
 */
@ParameterizedTest(name = "readVersionFromStoredRecordInTwoStores [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void readVersionFromStoredRecordInTwoStores(int testFormatVersion, boolean testSplitLongRecords) {
    formatVersion = testFormatVersion;
    splitLongRecords = testSplitLongRecords;
    final MySimpleRecord record1 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).build();
    final MySimpleRecord record2 = MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1729).build();
    final FDBRecordVersion version1;
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        FDBStoredRecord<?> storedRecord1 = recordStore.saveRecord(record1);
        assertNotNull(storedRecord1.getVersion());
        context.commit();
        assertNotNull(context.getVersionStamp());
        version1 = storedRecord1.getVersion().withCommittedVersion(context.getVersionStamp());
    }
    final FDBRecordVersion version2;
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).create();
        FDBStoredRecord<?> storedRecord2 = recordStore2.saveRecord(record2);
        assertNotNull(storedRecord2.getVersion());
        FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
        assertNotNull(storedRecord1);
        assertEquals(version1, storedRecord1.getVersion());
        context.commit();
        assertNotNull(context.getVersionStamp());
        version2 = storedRecord2.getVersion().withCommittedVersion(context.getVersionStamp());
    }
    try (FDBRecordContext context = openContext(simpleVersionHook)) {
        final FDBRecordStore recordStore2 = recordStore.asBuilder().setSubspace(subspace2).open();
        FDBStoredRecord<?> storedRecord1 = recordStore.loadRecord(Tuple.from(record1.getRecNo()));
        assertNotNull(storedRecord1);
        assertEquals(version1, storedRecord1.getVersion());
        assertEquals(record1, storedRecord1.getRecord());
        FDBStoredRecord<?> storedRecord2 = recordStore2.loadRecord(Tuple.from(record2.getRecNo()));
        assertNotNull(storedRecord2);
        assertEquals(version2, storedRecord2.getVersion());
        assertEquals(record2, storedRecord2.getRecord());
    }
}

Example 8 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreStateCacheTest method conflictWhenCachedChanged.

/**
 * Make sure that if one transaction changes the store header then an open store in another transaction that
 * loaded the store state from cache will fail at commit time with conflict.
 */
@ParameterizedTest(name = "conflictWhenCachedChanged (test context = {0})")
@MethodSource("testContextSource")
public void conflictWhenCachedChanged(@Nonnull StateCacheTestContext testContext) throws Exception {
    FDBRecordStoreStateCache origStoreStateCache = fdb.getStoreStateCache();
    try {
        fdb.setStoreStateCache(testContext.getCache(fdb));
        RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        metaDataBuilder.addIndex("MySimpleRecord", "num_value_2");
        RecordMetaData metaData2 = metaDataBuilder.getRecordMetaData();
        assertThat(metaData1.getVersion(), lessThan(metaData2.getVersion()));
        FDBRecordStore.Builder storeBuilder;
        // Initialize the record store with a meta-data store
        try (FDBRecordContext context = openContext()) {
            context.getTimer().reset();
            FDBRecordStore recordStore = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(metaData1).setKeySpacePath(path).create();
            assertEquals(1, context.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_MISS));
            assertEquals(metaData1.getVersion(), recordStore.getRecordStoreState().getStoreHeader().getMetaDataversion());
            commit(context);
            storeBuilder = recordStore.asBuilder();
        }
        // Load the record store state into the cache.
        try (FDBRecordContext context1 = testContext.getCachedContext(fdb, storeBuilder);
            FDBRecordContext context2 = testContext.getCachedContext(fdb, storeBuilder)) {
            context1.setTimer(new FDBStoreTimer());
            context2.setTimer(new FDBStoreTimer());
            FDBRecordStore recordStore1 = storeBuilder.copyBuilder().setContext(context1).setMetaDataProvider(metaData1).open();
            assertEquals(1, context1.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_HIT));
            assertEquals(metaData1.getVersion(), recordStore1.getRecordMetaData().getVersion());
            assertEquals(metaData1.getVersion(), recordStore1.getRecordStoreState().getStoreHeader().getMetaDataversion());
            // Update the meta-data in the second transaction
            FDBRecordStore recordStore2 = storeBuilder.copyBuilder().setContext(context2).setMetaDataProvider(metaData2).open();
            assertEquals(1, context2.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_HIT));
            assertEquals(Collections.singletonList(recordStore2.getRecordMetaData().getRecordType("MySimpleRecord")), recordStore2.getRecordMetaData().recordTypesForIndex(recordStore2.getRecordMetaData().getIndex("MySimpleRecord$num_value_2")));
            assertEquals(metaData2.getVersion(), recordStore2.getRecordMetaData().getVersion());
            assertEquals(metaData2.getVersion(), recordStore2.getRecordStoreState().getStoreHeader().getMetaDataversion());
            context2.commit();
            // Add a write to context1 so that the conflict ranges actually get checked.
            recordStore1.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066).setNumValue2(1415).build());
            // Should conflict on store header even though not actually read in this transaction
            assertThrows(FDBExceptions.FDBStoreTransactionConflictException.class, context1::commit);
        }
        // New transaction should now see the new meta-data version
        try (FDBRecordContext context = openContext()) {
            context.getTimer().reset();
            // Trying to load with the old meta-data should fail
            assertThrows(RecordStoreStaleMetaDataVersionException.class, () -> storeBuilder.copyBuilder().setContext(context).setMetaDataProvider(metaData1).open());
            assertEquals(1, context.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_MISS));
            // Trying to load with the new meta-data should succeed
            FDBRecordStore recordStore = storeBuilder.copyBuilder().setContext(context).setMetaDataProvider(metaData2).open();
            assertEquals(1, context.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_HIT));
            assertEquals(metaData2.getVersion(), recordStore.getRecordStoreState().getStoreHeader().getMetaDataversion());
        }
    } finally {
        fdb.setStoreStateCache(origStoreStateCache);
    }
}

Example 9 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreStateCacheTest method cacheTwoSubspaces.

/**
 * Make sure that caching two different subspaces are both cached but with separate entries.
 */
@ParameterizedTest(name = "cacheTwoSubspaces (test context = {0})")
@MethodSource("testContextSource")
public void cacheTwoSubspaces(@Nonnull StateCacheTestContext testContext) throws Exception {
    FDBRecordStoreStateCache origStoreStateCache = fdb.getStoreStateCache();
    try {
        fdb.setStoreStateCache(testContext.getCache(fdb));
        final KeySpacePath path1 = multiStorePath.add("storePath", "path1");
        final KeySpacePath path2 = multiStorePath.add("storePath", "path2");
        final FDBRecordStore.Builder storeBuilder1;
        final FDBRecordStore.Builder storeBuilder2;
        try (FDBRecordContext context = openContext()) {
            context.getTimer().reset();
            path1.deleteAllData(context);
            path2.deleteAllData(context);
            openSimpleRecordStore(context);
            FDBRecordStore store1 = recordStore.asBuilder().setKeySpacePath(path1).create();
            store1.setStateCacheabilityAsync(true).get();
            storeBuilder1 = store1.asBuilder();
            store1.markIndexWriteOnly("MySimpleRecord$str_value_indexed").get();
            FDBRecordStore store2 = recordStore.asBuilder().setKeySpacePath(path2).create();
            store2.setStateCacheabilityAsync(true).get();
            storeBuilder2 = store2.asBuilder();
            store2.markIndexDisabled("MySimpleRecord$num_value_3_indexed").get();
            commit(context);
        }
        // Open both paths. Only the one in path1 should be cached.
        long readVersion;
        try (FDBRecordContext context = testContext.getCachedContext(fdb, storeBuilder1, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS)) {
            FDBRecordStore store1 = storeBuilder1.setContext(context).open();
            assertEquals(1, context.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_HIT));
            assertTrue(store1.isIndexWriteOnly("MySimpleRecord$str_value_indexed"));
            assertTrue(store1.isIndexReadable("MySimpleRecord$num_value_3_indexed"));
            FDBRecordStore store2 = storeBuilder2.setContext(context).open();
            assertEquals(1, context.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_MISS));
            assertTrue(store2.isIndexReadable("MySimpleRecord$str_value_indexed"));
            assertTrue(store2.isIndexDisabled("MySimpleRecord$num_value_3_indexed"));
            readVersion = context.getReadVersion();
        }
        // Open both paths. Now they are both cached.
        try (FDBRecordContext context = openContext()) {
            context.getTimer().reset();
            context.setReadVersion(readVersion);
            FDBRecordStore store1 = storeBuilder1.setContext(context).open();
            assertEquals(1, context.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_HIT));
            assertTrue(store1.isIndexWriteOnly("MySimpleRecord$str_value_indexed"));
            assertTrue(store1.isIndexReadable("MySimpleRecord$num_value_3_indexed"));
            FDBRecordStore store2 = storeBuilder2.setContext(context).open();
            assertEquals(2, context.getTimer().getCount(FDBStoreTimer.Counts.STORE_STATE_CACHE_HIT));
            assertTrue(store2.isIndexReadable("MySimpleRecord$str_value_indexed"));
            assertTrue(store2.isIndexDisabled("MySimpleRecord$num_value_3_indexed"));
        }
    } finally {
        fdb.setStoreStateCache(origStoreStateCache);
    }
}

Example 10 with FDBRecordStore

use of com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore in project fdb-record-layer by FoundationDB.

the class SyntheticRecordPlannerTest method joinIndex.

@Test
public void joinIndex() throws Exception {
    metaDataBuilder.addIndex("MySimpleRecord", "other_rec_no");
    final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("Simple_Other");
    joined.addConstituent("simple", "MySimpleRecord");
    joined.addConstituent("other", "MyOtherRecord");
    joined.addJoin("simple", "other_rec_no", "other", "rec_no");
    metaDataBuilder.addIndex(joined, new Index("simple.str_value_other.num_value_3", concat(field("simple").nest("str_value"), field("other").nest("num_value_3"))));
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).create();
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < i; j++) {
                TestRecordsJoinIndexProto.MySimpleRecord.Builder simple = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder();
                simple.setRecNo(100 * i + j).setOtherRecNo(1000 + i);
                simple.setStrValue((i + j) % 2 == 0 ? "even" : "odd");
                recordStore.saveRecord(simple.build());
            }
            TestRecordsJoinIndexProto.MyOtherRecord.Builder other = TestRecordsJoinIndexProto.MyOtherRecord.newBuilder();
            other.setRecNo(1000 + i);
            other.setNumValue3(i);
            recordStore.saveRecord(other.build());
        }
        context.commit();
    }
    try (FDBRecordContext context = openContext()) {
        final FDBRecordStore recordStore = recordStoreBuilder.setContext(context).open();
        final Index index = recordStore.getRecordMetaData().getIndex("simple.str_value_other.num_value_3");
        final TupleRange range = new ScanComparisons.Builder().addEqualityComparison(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, "even")).addInequalityComparison(new Comparisons.SimpleComparison(Comparisons.Type.GREATER_THAN, 1)).build().toTupleRange();
        List<Tuple> expected1 = Arrays.asList(Tuple.from("even", 2, -1, Tuple.from(200), Tuple.from(1002)));
        List<Tuple> results1 = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).map(IndexEntry::getKey).asList().join();
        assertEquals(expected1, results1);
        FDBStoredRecord<Message> record = recordStore.loadRecord(Tuple.from(201));
        TestRecordsJoinIndexProto.MySimpleRecord.Builder recordBuilder = TestRecordsJoinIndexProto.MySimpleRecord.newBuilder().mergeFrom(record.getRecord());
        recordBuilder.setStrValue("even");
        recordStore.saveRecord(recordBuilder.build());
        List<Tuple> expected2 = Arrays.asList(Tuple.from("even", 2, -1, Tuple.from(200), Tuple.from(1002)), Tuple.from("even", 2, -1, Tuple.from(201), Tuple.from(1002)));
        List<Tuple> results2 = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).map(IndexEntry::getKey).asList().join();
        assertEquals(expected2, results2);
        recordStore.deleteRecord(Tuple.from(1002));
        List<Tuple> expected3 = Arrays.asList();
        List<Tuple> results3 = recordStore.scanIndex(index, IndexScanType.BY_VALUE, range, null, ScanProperties.FORWARD_SCAN).map(IndexEntry::getKey).asList().join();
        assertEquals(expected3, results3);
    }
}