Examples with KeySpacePath com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath used on opensource projects

Example 6 with KeySpacePath

use of com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreReplaceIndexTest method forEachStore.

private void forEachStore(@Nonnull KeySpacePath root, @Nonnull List<String> storePaths, @Nonnull BiConsumer<String, FDBRecordStore> subStoreConsumer) {
    for (String storePathName : storePaths) {
        final KeySpacePath storePath = root.add("storePath", storePathName);
        final FDBRecordStore subStore = recordStore.asBuilder().setKeySpacePath(storePath).createOrOpen();
        subStoreConsumer.accept(storePathName, subStore);
    }
}

Example 7 with KeySpacePath

use of com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method open.

@Test
public void open() throws Exception {
    // This tests the functionality of "open", so doesn't use the same method of opening
    // the record store that other methods within this class use.
    Object[] metaDataPathObjects = new Object[] { "record-test", "unit", "metadataStore" };
    KeySpacePath metaDataPath;
    Subspace expectedSubspace;
    Subspace metaDataSubspace;
    try (FDBRecordContext context = fdb.openContext()) {
        metaDataPath = TestKeySpace.getKeyspacePath(metaDataPathObjects);
        expectedSubspace = path.toSubspace(context);
        metaDataSubspace = metaDataPath.toSubspace(context);
        context.ensureActive().clear(Range.startsWith(metaDataSubspace.pack()));
        context.commit();
    }
    Index newIndex = new Index("newIndex", concatenateFields("str_value_indexed", "num_value_3_indexed"));
    Index newIndex2 = new Index("newIndex2", concatenateFields("str_value_indexed", "rec_no"));
    TestRecords1Proto.MySimpleRecord record = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).setStrValueIndexed("value").setNumValue3Indexed(1729).build();
    try (FDBRecordContext context = fdb.openContext()) {
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        FDBRecordStore recordStore = storeBuilder(context, metaDataBuilder).createOrOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(metaDataBuilder.getVersion(), recordStore.getRecordMetaData().getVersion());
        final int version = metaDataBuilder.getVersion();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        recordStore = recordStore.asBuilder().setMetaDataProvider(metaDataBuilder).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        // This stops the index build.
        recordStore.saveRecord(record);
        final RecordMetaData staleMetaData = metaDataBuilder.getRecordMetaData();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        recordStore = recordStore.asBuilder().setMetaDataProvider(metaDataBuilder).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertEquals(Collections.singleton(newIndex2.getName()), recordStore.getRecordStoreState().getDisabledIndexNames());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        final FDBRecordStore.Builder staleBuilder = recordStore.asBuilder().setMetaDataProvider(staleMetaData);
        TestHelpers.assertThrows(RecordStoreStaleMetaDataVersionException.class, staleBuilder::createOrOpen, LogMessageKeys.LOCAL_VERSION.toString(), version + 1, LogMessageKeys.STORED_VERSION.toString(), version + 2);
    }
    try (FDBRecordContext context = fdb.openContext()) {
        FDBMetaDataStore metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, null);
        FDBRecordStore.newBuilder().setMetaDataStore(metaDataStore).setContext(context).setKeySpacePath(path).createOrOpenAsync().handle((store, e) -> {
            assertNull(store);
            assertNotNull(e);
            assertThat(e, instanceOf(CompletionException.class));
            Throwable cause = e.getCause();
            assertNotNull(cause);
            assertThat(cause, instanceOf(FDBMetaDataStore.MissingMetaDataException.class));
            return null;
        }).join();
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        RecordMetaData origMetaData = metaDataBuilder.getRecordMetaData();
        final int version = origMetaData.getVersion();
        FDBRecordStore recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).createOrOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version, recordStore.getRecordMetaData().getVersion());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        metaDataStore.saveAndSetCurrent(metaDataBuilder.getRecordMetaData().toProto()).join();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        recordStore.saveRecord(record);
        final FDBMetaDataStore staleMetaDataStore = metaDataStore;
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        metaDataStore.saveRecordMetaData(metaDataBuilder.getRecordMetaData());
        recordStore = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(metaDataStore).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertEquals(Collections.singleton(newIndex2.getName()), recordStore.getRecordStoreState().getDisabledIndexNames());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        // The stale meta-data store uses the cached meta-data, hence the stale version exception
        FDBRecordStore.Builder storeBuilder = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(staleMetaDataStore);
        TestHelpers.assertThrows(RecordStoreStaleMetaDataVersionException.class, storeBuilder::createOrOpen, LogMessageKeys.LOCAL_VERSION.toString(), version + 1, LogMessageKeys.STORED_VERSION.toString(), version + 2);
    }
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        FDBRecordStore recordStore = storeBuilder(context, metaDataBuilder).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(metaDataBuilder.getVersion(), recordStore.getRecordMetaData().getVersion());
        final int version = metaDataBuilder.getVersion();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        recordStore = recordStore.asBuilder().setMetaDataProvider(metaDataBuilder).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        // This would stop the build if this ran checkVersion.
        recordStore.saveRecord(record);
        final RecordMetaData staleMetaData = metaDataBuilder.getRecordMetaData();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        recordStore = storeBuilder(context, metaDataBuilder).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        recordStore = recordStore.asBuilder().setMetaDataProvider(staleMetaData).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
    }
    try (FDBRecordContext context = fdb.openContext()) {
        FDBMetaDataStore metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, null);
        FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataStore(metaDataStore).uncheckedOpenAsync().handle((store, e) -> {
            assertNull(store);
            assertNotNull(e);
            assertThat(e, instanceOf(CompletionException.class));
            Throwable cause = e.getCause();
            assertNotNull(cause);
            assertThat(cause, instanceOf(FDBMetaDataStore.MissingMetaDataException.class));
            return null;
        }).join();
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        RecordMetaData origMetaData = metaDataBuilder.getRecordMetaData();
        int version = origMetaData.getVersion();
        FDBRecordStore recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version, recordStore.getRecordMetaData().getVersion());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        metaDataStore.saveAndSetCurrent(metaDataBuilder.getRecordMetaData().toProto()).join();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        recordStore = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(metaDataStore).setMetaDataProvider(origMetaData).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        // This would stop the build if this used checkVersion
        recordStore.saveRecord(record);
        final FDBMetaDataStore staleMetaDataStore = metaDataStore;
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        metaDataStore.saveAndSetCurrent(metaDataBuilder.getRecordMetaData().toProto()).join();
        recordStore = FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataStore(metaDataStore).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        // The stale meta-data store uses the cached meta-data, hence the old version in the final assert
        recordStore = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(staleMetaDataStore).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
    }
}

Example 8 with KeySpacePath

use of com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath in project fdb-record-layer by FoundationDB.

the class FDBRecordStorePerformanceTest method populate.

public void populate() {
    int n = 0;
    while (n < databaseParameters.recordCount) {
        try (FDBRecordContext context = fdb.openContext()) {
            final KeySpacePath keyspacePath = TestKeySpace.getKeyspacePath(databaseParameters.path);
            final FDBRecordStore.Builder storeBuilder = FDBRecordStore.newBuilder().setContext(context).setMetaDataProvider(metaData).setKeySpacePath(keyspacePath);
            final FDBRecordStore recordStore;
            if (n == 0) {
                FDBRecordStore.deleteStore(context, keyspacePath);
                recordStore = storeBuilder.create();
            } else {
                recordStore = storeBuilder.open();
            }
            int c = 0;
            while (c < databaseParameters.recordsPerCommit && n < databaseParameters.recordCount) {
                TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
                recBuilder.setRecNo(n);
                if (databaseParameters.stringSize > 0) {
                    recBuilder.setStrValueIndexed(StringUtils.repeat((char) ('A' + n % 26), databaseParameters.stringSize));
                }
                if (databaseParameters.rankIndex) {
                    recBuilder.setNumValueUnique(n);
                }
                recBuilder.setNumValue2((n + databaseParameters.recordsPerBucket) % databaseParameters.recordCount);
                recBuilder.setNumValue3Indexed(n / databaseParameters.recordsPerBucket);
                recordStore.saveRecord(recBuilder.build());
                n++;
                c++;
            }
            context.commit();
        }
    }
}

Example 9 with KeySpacePath

use of com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreReplaceIndexTest method buildReplacementsInMultipleStores.

@Test
public void buildReplacementsInMultipleStores() {
    final Object[] multiStorePathObjects = { "record-test", "unit", "multiRecordStore" };
    final KeySpacePath multiStoreRoot = TestKeySpace.getKeyspacePath(multiStorePathObjects);
    try {
        final String recordTypeName = "MySimpleRecord";
        final Index origIndex = new Index("MySimpleRecord$repeater", Key.Expressions.field("repeater", KeyExpression.FanType.FanOut));
        final Index newIndex = new Index("MySimpleRecord$(num_value_2, repeater)", Key.Expressions.concat(Key.Expressions.field("num_value_2"), Key.Expressions.field("repeater", KeyExpression.FanType.FanOut)));
        final RecordMetaDataHook allIndexesHook = composeHooks(addIndexHook(recordTypeName, origIndex), addIndexHook(recordTypeName, newIndex));
        final List<String> stores = IntStream.range(0, 10).mapToObj(i -> "store_" + i).collect(Collectors.toList());
        try (FDBRecordContext context = openContext()) {
            multiStoreRoot.deleteAllData(context);
            openSimpleRecordStore(context, allIndexesHook);
            // Create a bunch of stores and disable the new index in all of them
            forEachStore(multiStoreRoot, stores, (storePathName, subStore) -> {
                assertTrue(context.asyncToSync(FDBStoreTimer.Waits.WAIT_DROP_INDEX, subStore.markIndexDisabled(newIndex)));
                subStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).addRepeater(42).addRepeater(800).setStrValueIndexed(storePathName).build());
            });
            commit(context);
        }
        final RecordMetaDataHook withReplacementHook = composeHooks(addIndexAndReplacements(recordTypeName, origIndex, newIndex), bumpMetaDataVersionHook());
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context, withReplacementHook);
            forEachStore(multiStoreRoot, stores, (storePathName, subStore) -> {
                final List<FDBIndexedRecord<Message>> records = context.asyncToSync(FDBStoreTimer.Waits.WAIT_SCAN_RECORDS, subStore.scanIndexRecords(origIndex.getName()).asList());
                assertThat(records, hasSize(2));
                records.stream().map(FDBIndexedRecord::getRecord).map(msg -> msg.getField(msg.getDescriptorForType().findFieldByName("str_value_indexed"))).map(fieldValue -> {
                    assertThat(fieldValue, instanceOf(String.class));
                    return (String) fieldValue;
                }).forEach(strValue -> assertEquals(storePathName, strValue));
                context.asyncToSync(FDBStoreTimer.Waits.WAIT_ONLINE_BUILD_INDEX, subStore.rebuildIndex(subStore.getRecordMetaData().getIndex(newIndex.getName())));
            });
            commit(context);
        }
        // Validate that each store has had the original index removed (because the replacement index was built)
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context, withReplacementHook);
            forEachStore(multiStoreRoot, stores, (storePathName, subStore) -> assertTrue(subStore.isIndexDisabled(origIndex.getName())));
            commit(context);
        }
        // Validate each store has had the old index data cleaned out
        try (FDBRecordContext context = openContext()) {
            openSimpleRecordStore(context, composeHooks(allIndexesHook, bumpMetaDataVersionHook(), bumpMetaDataVersionHook()));
            forEachStore(multiStoreRoot, stores, (storePathName, subStore) -> {
                assertTrue(context.asyncToSync(FDBStoreTimer.Waits.WAIT_ADD_INDEX, subStore.uncheckedMarkIndexReadable(origIndex.getName())));
                assertEquals(Collections.emptyList(), context.asyncToSync(FDBStoreTimer.Waits.WAIT_SCAN_INDEX_RECORDS, subStore.scanIndexRecords(origIndex.getName()).asList()));
            });
        }
    } finally {
        try (FDBRecordContext context = openContext()) {
            multiStoreRoot.deleteAllData(context);
            commit(context);
        }
    }
}

Example 10 with KeySpacePath

use of com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath in project SBK by kmgowda.

the class FdbRecord method openStorage.

@Override
public void openStorage(final ParameterOptions params) throws IOException {
    db = FDBDatabaseFactory.instance().getDatabase(config.cFile);
    // Define the keyspace for our application
    KeySpace keySpace = new KeySpace(new KeySpaceDirectory(config.keySpace, KeySpaceDirectory.KeyType.STRING, config.keySpace));
    // Get the path where our record store will be rooted
    KeySpacePath path = keySpace.path(config.keySpace);
    RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(FdbRecordLayerProto.getDescriptor());
    metaDataBuilder.getRecordType("Record").setPrimaryKey(Key.Expressions.field("recordID"));
    recordStoreProvider = context -> FDBRecordStore.newBuilder().setMetaDataProvider(metaDataBuilder).setContext(context).setKeySpacePath(path).createOrOpen();
    if (params.getWritersCount() > 0) {
        db.run(context -> {
            FDBRecordStore recordStore = recordStoreProvider.apply(context);
            recordStore.deleteAllRecords();
            return null;
        });
    }
}