Examples with Subspace com.apple.foundationdb.subspace.Subspace used on opensource projects

Example 36 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class FDBRecordContextTest method logTransactionAfterGettingAReadVersion.

@Test
public void logTransactionAfterGettingAReadVersion() {
    try (FDBRecordContext context = fdb.openContext(null, null, null, FDBTransactionPriority.DEFAULT, "logTransactionAfterGrv")) {
        context.ensureActive().getReadVersion().join();
        context.logTransaction();
        Subspace fakeSubspace = new Subspace(Tuple.from(UUID.randomUUID()));
        context.ensureActive().addWriteConflictRange(fakeSubspace.range().begin, fakeSubspace.range().end);
        context.commit();
    }
}

Example 37 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method open.

@Test
public void open() throws Exception {
    // This tests the functionality of "open", so doesn't use the same method of opening
    // the record store that other methods within this class use.
    Object[] metaDataPathObjects = new Object[] { "record-test", "unit", "metadataStore" };
    KeySpacePath metaDataPath;
    Subspace expectedSubspace;
    Subspace metaDataSubspace;
    try (FDBRecordContext context = fdb.openContext()) {
        metaDataPath = TestKeySpace.getKeyspacePath(metaDataPathObjects);
        expectedSubspace = path.toSubspace(context);
        metaDataSubspace = metaDataPath.toSubspace(context);
        context.ensureActive().clear(Range.startsWith(metaDataSubspace.pack()));
        context.commit();
    }
    Index newIndex = new Index("newIndex", concatenateFields("str_value_indexed", "num_value_3_indexed"));
    Index newIndex2 = new Index("newIndex2", concatenateFields("str_value_indexed", "rec_no"));
    TestRecords1Proto.MySimpleRecord record = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).setStrValueIndexed("value").setNumValue3Indexed(1729).build();
    try (FDBRecordContext context = fdb.openContext()) {
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        FDBRecordStore recordStore = storeBuilder(context, metaDataBuilder).createOrOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(metaDataBuilder.getVersion(), recordStore.getRecordMetaData().getVersion());
        final int version = metaDataBuilder.getVersion();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        recordStore = recordStore.asBuilder().setMetaDataProvider(metaDataBuilder).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        // This stops the index build.
        recordStore.saveRecord(record);
        final RecordMetaData staleMetaData = metaDataBuilder.getRecordMetaData();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        recordStore = recordStore.asBuilder().setMetaDataProvider(metaDataBuilder).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertEquals(Collections.singleton(newIndex2.getName()), recordStore.getRecordStoreState().getDisabledIndexNames());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        final FDBRecordStore.Builder staleBuilder = recordStore.asBuilder().setMetaDataProvider(staleMetaData);
        TestHelpers.assertThrows(RecordStoreStaleMetaDataVersionException.class, staleBuilder::createOrOpen, LogMessageKeys.LOCAL_VERSION.toString(), version + 1, LogMessageKeys.STORED_VERSION.toString(), version + 2);
    }
    try (FDBRecordContext context = fdb.openContext()) {
        FDBMetaDataStore metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, null);
        FDBRecordStore.newBuilder().setMetaDataStore(metaDataStore).setContext(context).setKeySpacePath(path).createOrOpenAsync().handle((store, e) -> {
            assertNull(store);
            assertNotNull(e);
            assertThat(e, instanceOf(CompletionException.class));
            Throwable cause = e.getCause();
            assertNotNull(cause);
            assertThat(cause, instanceOf(FDBMetaDataStore.MissingMetaDataException.class));
            return null;
        }).join();
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        RecordMetaData origMetaData = metaDataBuilder.getRecordMetaData();
        final int version = origMetaData.getVersion();
        FDBRecordStore recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).createOrOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version, recordStore.getRecordMetaData().getVersion());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        metaDataStore.saveAndSetCurrent(metaDataBuilder.getRecordMetaData().toProto()).join();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        recordStore.saveRecord(record);
        final FDBMetaDataStore staleMetaDataStore = metaDataStore;
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        metaDataStore.saveRecordMetaData(metaDataBuilder.getRecordMetaData());
        recordStore = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(metaDataStore).open();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertEquals(recordStore.getRecordStoreState(), recordStore.getRecordStoreState());
        assertEquals(Collections.singleton(newIndex2.getName()), recordStore.getRecordStoreState().getDisabledIndexNames());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        // The stale meta-data store uses the cached meta-data, hence the stale version exception
        FDBRecordStore.Builder storeBuilder = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(staleMetaDataStore);
        TestHelpers.assertThrows(RecordStoreStaleMetaDataVersionException.class, storeBuilder::createOrOpen, LogMessageKeys.LOCAL_VERSION.toString(), version + 1, LogMessageKeys.STORED_VERSION.toString(), version + 2);
    }
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        FDBRecordStore recordStore = storeBuilder(context, metaDataBuilder).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(metaDataBuilder.getVersion(), recordStore.getRecordMetaData().getVersion());
        final int version = metaDataBuilder.getVersion();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        recordStore = recordStore.asBuilder().setMetaDataProvider(metaDataBuilder).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        // This would stop the build if this ran checkVersion.
        recordStore.saveRecord(record);
        final RecordMetaData staleMetaData = metaDataBuilder.getRecordMetaData();
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        recordStore = storeBuilder(context, metaDataBuilder).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        recordStore = recordStore.asBuilder().setMetaDataProvider(staleMetaData).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
    }
    try (FDBRecordContext context = fdb.openContext()) {
        FDBMetaDataStore metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, null);
        FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataStore(metaDataStore).uncheckedOpenAsync().handle((store, e) -> {
            assertNull(store);
            assertNotNull(e);
            assertThat(e, instanceOf(CompletionException.class));
            Throwable cause = e.getCause();
            assertNotNull(cause);
            assertThat(cause, instanceOf(FDBMetaDataStore.MissingMetaDataException.class));
            return null;
        }).join();
        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
        RecordMetaData origMetaData = metaDataBuilder.getRecordMetaData();
        int version = origMetaData.getVersion();
        FDBRecordStore recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version, recordStore.getRecordMetaData().getVersion());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex);
        metaDataStore.saveAndSetCurrent(metaDataBuilder.getRecordMetaData().toProto()).join();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        recordStore = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(metaDataStore).setMetaDataProvider(origMetaData).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
        // This would stop the build if this used checkVersion
        recordStore.saveRecord(record);
        final FDBMetaDataStore staleMetaDataStore = metaDataStore;
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        metaDataBuilder.addIndex("MySimpleRecord", newIndex2);
        metaDataStore.saveAndSetCurrent(metaDataBuilder.getRecordMetaData().toProto()).join();
        recordStore = FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataStore(metaDataStore).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        // The stale meta-data store uses the cached meta-data, hence the old version in the final assert
        recordStore = FDBRecordStore.newBuilder().setContext(context).setSubspace(expectedSubspace).setMetaDataStore(staleMetaDataStore).uncheckedOpen();
        assertEquals(expectedSubspace, recordStore.getSubspace());
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable());
        assertEquals(version + 1, recordStore.getRecordMetaData().getVersion());
    }
}

Example 38 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class SplitHelperTest method saveSuccessfully.

private SplitHelper.SizeInfo saveSuccessfully(@Nonnull FDBRecordContext context, @Nonnull Tuple key, byte[] serialized, @Nullable FDBRecordVersion version, boolean splitLongRecords, boolean omitUnsplitSuffix, @Nullable FDBStoredSizes previousSizeInfo) {
    final SplitHelper.SizeInfo sizeInfo = new SplitHelper.SizeInfo();
    SplitHelper.saveWithSplit(context, subspace, key, serialized, version, splitLongRecords, omitUnsplitSuffix, previousSizeInfo != null, previousSizeInfo, sizeInfo);
    int dataKeyCount = (serialized.length - 1) / SplitHelper.SPLIT_RECORD_SIZE + 1;
    boolean isSplit = dataKeyCount > 1;
    int keyCount = dataKeyCount;
    if (version != null) {
        keyCount += 1;
    }
    int keySize = (subspace.pack().length + key.pack().length) * keyCount;
    assertEquals(isSplit, sizeInfo.isSplit());
    assertEquals(keyCount, sizeInfo.getKeyCount());
    if (!omitUnsplitSuffix || splitLongRecords) {
        // Add in the the counters the split points.
        if (!isSplit) {
            // As 0 requires 1 byte when Tuple packed
            keySize += 1;
        } else {
            // As each split point is two bytes when tuple packed
            keySize += dataKeyCount * 2;
        }
    }
    if (version != null) {
        keySize += 2;
    }
    int valueSize = serialized.length + (version != null ? 1 + FDBRecordVersion.VERSION_LENGTH : 0);
    assertEquals(keySize, sizeInfo.getKeySize());
    assertEquals(valueSize, sizeInfo.getValueSize());
    assertEquals(version != null, sizeInfo.isVersionedInline());
    final Subspace keySubspace = subspace.subspace(key);
    RecordCursorIterator<KeyValue> kvCursor = KeyValueCursor.Builder.withSubspace(keySubspace).setContext(context).setScanProperties(ScanProperties.FORWARD_SCAN).build().asIterator();
    List<Long> indexes = new ArrayList<>(keyCount);
    byte[] versionBytes = null;
    byte[] valueBytes = null;
    while (kvCursor.hasNext()) {
        KeyValue kv = kvCursor.next();
        Tuple suffix = keySubspace.unpack(kv.getKey());
        if (omitUnsplitSuffix) {
            assertThat(suffix.isEmpty(), is(true));
            valueBytes = kv.getValue();
        } else {
            Long index = suffix.getLong(0);
            indexes.add(index);
            if (index == SplitHelper.RECORD_VERSION) {
                versionBytes = kv.getValue();
            } else {
                if (valueBytes == null) {
                    valueBytes = kv.getValue();
                } else {
                    valueBytes = ByteArrayUtil.join(valueBytes, kv.getValue());
                }
            }
        }
    }
    List<Long> expectedIndexes;
    if (omitUnsplitSuffix) {
        expectedIndexes = Collections.emptyList();
    } else {
        expectedIndexes = new ArrayList<>(keyCount);
        if (version != null && version.isComplete()) {
            expectedIndexes.add(SplitHelper.RECORD_VERSION);
        }
        if (!isSplit) {
            expectedIndexes.add(SplitHelper.UNSPLIT_RECORD);
        } else {
            LongStream.range(SplitHelper.START_SPLIT_RECORD, SplitHelper.START_SPLIT_RECORD + dataKeyCount).forEach(expectedIndexes::add);
        }
    }
    assertEquals(expectedIndexes, indexes);
    assertNotNull(valueBytes);
    assertArrayEquals(serialized, valueBytes);
    if (version != null) {
        if (!version.isComplete()) {
            assertNull(versionBytes);
        } else {
            assertNotNull(versionBytes);
            assertEquals(version, FDBRecordVersion.fromVersionstamp(Tuple.fromBytes(versionBytes).getVersionstamp(0)));
        }
    } else {
        assertNull(versionBytes);
    }
    return sizeInfo;
}

Example 39 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class TextIndexTest method printUsage.

private void printUsage() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        Subspace indexSubspace = recordStore.getIndexMaintainer(recordStore.getRecordMetaData().getIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME)).getIndexSubspace();
        final int indexSuspaceLength = indexSubspace.getKey().length;
        int subspaceOverhead = 0;
        int keySize = 0;
        int valueSize = 0;
        for (KeyValue kv : context.ensureActive().getRange(indexSubspace.range())) {
            subspaceOverhead += indexSuspaceLength;
            keySize += kv.getKey().length - indexSuspaceLength;
            valueSize += kv.getValue().length;
        }
        int textSize = 0;
        RecordCursorIterator<String> cursor = recordStore.scanRecords(null, ScanProperties.FORWARD_SCAN).map(record -> {
            Message msg = record.getRecord();
            Descriptors.FieldDescriptor fd = msg.getDescriptorForType().findFieldByName("text");
            return msg.getField(fd).toString();
        }).asIterator();
        while (cursor.hasNext()) {
            textSize += cursor.next().length();
        }
        LOGGER.info("Usage:");
        LOGGER.info("  Subspace: {} kB", subspaceOverhead * 1e-3);
        LOGGER.info("  Keys:     {} kB", keySize * 1e-3);
        LOGGER.info("  Values:   {} kB", valueSize * 1e-3);
        LOGGER.info("  Text:     {} kB", textSize * 1e-3);
        LOGGER.info("  Overhead: {}", textSize == 0.0 ? Double.POSITIVE_INFINITY : ((subspaceOverhead + keySize + valueSize) * 1.0 / textSize));
    }
}

Example 40 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method validateUsingNewerVersionFormat.

private <M extends Message> void validateUsingNewerVersionFormat(@Nonnull List<FDBStoredRecord<M>> storedRecords) {
    // Make sure the old keyspace doesn't have anything in it
    final Subspace legacyVersionSubspace = recordStore.getLegacyVersionSubspace();
    KeyValueCursor legacyKvs = KeyValueCursor.Builder.withSubspace(legacyVersionSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build();
    assertEquals(0, (int) legacyKvs.getCount().join());
    // Look for the versions within the primary keyspace
    final Subspace recordsSubspace = recordStore.recordsSubspace();
    RecordCursorIterator<Pair<Tuple, FDBRecordVersion>> versionKeyPairs = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().map(kv -> Pair.of(recordsSubspace.unpack(kv.getKey()), kv.getValue())).filter(tupleBytesPair -> tupleBytesPair.getLeft().getLong(tupleBytesPair.getLeft().size() - 1) == -1).map(tupleBytesPair -> Pair.of(tupleBytesPair.getLeft().popBack(), FDBRecordVersion.fromVersionstamp(Tuple.fromBytes(tupleBytesPair.getRight()).getVersionstamp(0)))).asIterator();
    for (FDBStoredRecord<M> storedRecord : storedRecords) {
        assertTrue(versionKeyPairs.hasNext());
        Pair<Tuple, FDBRecordVersion> versionPair = versionKeyPairs.next();
        assertEquals(storedRecord.getPrimaryKey(), versionPair.getLeft());
        assertEquals(storedRecord.getVersion(), versionPair.getRight());
    }
    assertFalse(versionKeyPairs.hasNext());
}