Examples with Subspace com.apple.foundationdb.subspace.Subspace used on opensource projects

Example 26 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class VersionIndexTest method validateUsingOlderVersionFormat.

private <M extends Message> void validateUsingOlderVersionFormat(@Nonnull List<FDBStoredRecord<M>> storedRecords) {
    // Make sure all of the records have versions in the old keyspace
    final Subspace legacyVersionSubspace = recordStore.getLegacyVersionSubspace();
    RecordCursorIterator<Pair<Tuple, FDBRecordVersion>> versionKeyPairs = KeyValueCursor.Builder.withSubspace(legacyVersionSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().map(kv -> Pair.of(legacyVersionSubspace.unpack(kv.getKey()), FDBRecordVersion.fromBytes(kv.getValue()))).asIterator();
    for (FDBStoredRecord<M> storedRecord : storedRecords) {
        assertTrue(versionKeyPairs.hasNext());
        Pair<Tuple, FDBRecordVersion> versionPair = versionKeyPairs.next();
        assertEquals(storedRecord.getPrimaryKey(), versionPair.getLeft());
        assertEquals(storedRecord.getVersion(), versionPair.getRight());
    }
    assertFalse(versionKeyPairs.hasNext());
    // Validate that no value in the record subspace begins with the type code for versionstamps
    final Subspace recordsSubspace = recordStore.recordsSubspace();
    KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().forEach(kv -> assertNotEquals(VERSIONSTAMP_CODE, kv.getValue()[0])).join();
}

Example 27 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method storeExistenceChecksWithNoRecords.

@Test
public void storeExistenceChecksWithNoRecords() throws Exception {
    RecordMetaData metaData = RecordMetaData.build(TestRecords1Proto.getDescriptor());
    FDBRecordStore.Builder storeBuilder;
    try (FDBRecordContext context = openContext()) {
        storeBuilder = storeBuilder(context, metaData);
        FDBRecordStore store = storeBuilder.create();
        // delete the header
        store.ensureContextActive().clear(getStoreInfoKey(store));
        commit(context);
    }
    // Should be able to recover from a completely empty record store
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        FDBRecordStore store = storeBuilder.createOrOpen();
        // put a range subspace in for an index so that a future store opening can see it
        store.ensureContextActive().clear(getStoreInfoKey(store));
        Index foundIndex = metaData.getAllIndexes().stream().findAny().orElseGet(() -> fail("no indexes defined in meta-data"));
        new RangeSet(store.indexRangeSubspace(foundIndex)).insertRange(context.ensureActive(), null, null).get();
        // re-delete the header
        store.ensureContextActive().clear(getStoreInfoKey(store));
        commit(context);
    }
    // should be recoverable using ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        // do not perform checkVersion yet
        FDBRecordStore store = storeBuilder.build();
        assertNull(context.ensureActive().get(getStoreInfoKey(store)).get());
        assertTrue(store.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES).get());
        commit(context);
    }
    // Delete everything except a value in the index build space
    try (FDBRecordContext context = openContext()) {
        FDBRecordStore store = storeBuilder.setContext(context).open();
        final Subspace subspace = OnlineIndexer.indexBuildScannedRecordsSubspace(store, metaData.getIndex("MySimpleRecord$str_value_indexed"));
        // set a key in the INDEX_BUILD_SPACE
        context.ensureActive().set(subspace.getKey(), FDBRecordStore.encodeRecordCount(1215));
        context.ensureActive().clear(store.getSubspace().getKey(), subspace.getKey());
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context).createOrOpen(FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES);
        commit(context);
    }
    // Insert a record, then delete the store header
    try (FDBRecordContext context = openContext()) {
        // open as the previous open with the relaxed existence check should have fixed the store header
        FDBRecordStore store = storeBuilder.setContext(context).open();
        store.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).build());
        store.ensureContextActive().clear(getStoreInfoKey(store));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, () -> storeBuilder.createOrOpen(FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES));
        commit(context);
    }
    // Delete the record store, then insert a key at an unknown keyspace
    try (FDBRecordContext context = openContext()) {
        FDBRecordStore.deleteStore(context, path);
        Subspace subspace = path.toSubspace(context);
        context.ensureActive().set(subspace.pack("unknown_keyspace"), Tuple.from("doesn't matter").pack());
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
        RecordCoreException err = assertThrows(RecordCoreException.class, () -> storeBuilder.createOrOpen(FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES));
        assertEquals("Unrecognized keyspace: unknown_keyspace", err.getMessage());
        commit(context);
    }
}

Example 28 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method testUpdateRecords.

@Test
public void testUpdateRecords() {
    KeySpacePath metaDataPath;
    Subspace metaDataSubspace;
    try (FDBRecordContext context = fdb.openContext()) {
        metaDataPath = TestKeySpace.getKeyspacePath("record-test", "unit", "metadataStore");
        metaDataSubspace = metaDataPath.toSubspace(context);
        context.ensureActive().clear(Range.startsWith(metaDataSubspace.pack()));
        context.commit();
    }
    try (FDBRecordContext context = fdb.openContext()) {
        RecordMetaData origMetaData = RecordMetaData.build(TestRecords1Proto.getDescriptor());
        final int version = origMetaData.getVersion();
        FDBMetaDataStore metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        FDBRecordStore recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).createOrOpen();
        assertEquals(version, recordStore.getRecordMetaData().getVersion());
        TestRecords1Proto.MySimpleRecord record = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).setStrValueIndexed("value").setNumValue3Indexed(1729).build();
        recordStore.saveRecord(record);
        // Update the records without a local descriptor. Storing an evolved record must fail.
        final TestRecords1EvolvedProto.MySimpleRecord evolvedRecord = TestRecords1EvolvedProto.MySimpleRecord.newBuilder().setRecNo(1067L).setNumValue2(43).setStrValueIndexed("evolved value").setNumValue3Indexed(1730).build();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, null);
        // Bumps the version
        metaDataStore.updateRecords(TestRecords1EvolvedProto.getDescriptor());
        final FDBRecordStore recordStoreWithNoLocalFileDescriptor = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(version + 1, recordStoreWithNoLocalFileDescriptor.getRecordMetaData().getVersion());
        MetaDataException e = assertThrows(MetaDataException.class, () -> recordStoreWithNoLocalFileDescriptor.saveRecord(evolvedRecord));
        assertEquals(e.getMessage(), "descriptor did not match record type");
        // Update the records with a local descriptor. Storing an evolved record must succeed this time.
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1EvolvedProto.getDescriptor());
        // Bumps the version
        metaDataStore.updateRecords(TestRecords1EvolvedProto.getDescriptor());
        recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        recordStore.saveRecord(evolvedRecord);
        // Evolve the meta-data one more time and use it for local file descriptor. SaveRecord will succeed.
        final TestRecords1EvolvedAgainProto.MySimpleRecord evolvedAgainRecord = TestRecords1EvolvedAgainProto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).setStrValueIndexed("value").setNumValue3Indexed(1729).build();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1EvolvedAgainProto.getDescriptor());
        // Bumps the version
        metaDataStore.updateRecords(TestRecords1EvolvedProto.getDescriptor());
        recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(version + 3, recordStore.getRecordMetaData().getVersion());
        recordStore.saveRecord(evolvedAgainRecord);
    }
}

Example 29 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class SizeStatisticsCollectorTest method indexSize.

@Test
public void indexSize() throws Exception {
    final int recordCount = 100;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final Subspace indexSubspace = recordStore.indexSubspace(recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed"));
        final int indexSubspaceSize = indexSubspace.pack().length;
        long[] sizeBuckets = new long[Integer.SIZE];
        List<Integer> keySizes = new ArrayList<>(recordCount);
        int keySize = 0;
        for (int i = 0; i < recordCount; i++) {
            MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().setRecNo(i).setStrValueIndexed(Strings.repeat("x", i)).build();
            recordStore.saveRecord(simpleRecord);
            // Size contributions from:
            // index prefix      + index key (+ overhead) + primary key
            int indexKeySize = indexSubspaceSize + i + 2 + Tuple.from(i).pack().length;
            keySize += indexKeySize;
            int msb = Integer.SIZE - Integer.numberOfLeadingZeros(indexKeySize) - 1;
            sizeBuckets[msb] += 1;
            keySizes.add(indexKeySize);
        }
        SizeStatisticsCollector indexCollector = new SizeStatisticsCollector(recordStore, "MySimpleRecord$str_value_indexed");
        assertThat(indexCollector.collect(context, ExecuteProperties.SERIAL_EXECUTE), is(true));
        assertEquals(keySize, indexCollector.getKeySize());
        assertEquals(0, indexCollector.getValueSize());
        assertEquals(keySize, indexCollector.getTotalSize());
        assertEquals(keySize / (1.0 * recordCount), indexCollector.getAverage());
        assertArrayEquals(sizeBuckets, indexCollector.getSizeBuckets());
        for (double proportion : Arrays.asList(0.2, 0.5, 0.75, 0.90, 0.95)) {
            int realValue = keySizes.get((int) (keySizes.size() * proportion));
            int lowerBound = 1 << (Integer.SIZE - Integer.numberOfLeadingZeros(realValue) - 1);
            int upperBound = 1 << (Integer.SIZE - Integer.numberOfLeadingZeros(realValue));
            assertThat(indexCollector.getProportion(proportion), allOf(lessThanOrEqualTo((double) upperBound), greaterThanOrEqualTo((double) lowerBound)));
        }
        assertEquals(indexCollector.getProportion(0.5), indexCollector.getMedian());
        assertEquals(indexCollector.getProportion(0.90), indexCollector.getP90());
        assertEquals(indexCollector.getProportion(0.95), indexCollector.getP95());
        commit(context);
    }
}

Example 30 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class OnlineIndexScrubberTest method testScrubberSimpleDangling.

@Test
void testScrubberSimpleDangling() throws ExecutionException, InterruptedException {
    final FDBStoreTimer timer = new FDBStoreTimer();
    long numRecords = 51;
    long res;
    Index tgtIndex = new Index("tgt_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(tgtIndex);
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildIndex(tgtIndex);
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).setScrubbingPolicy(OnlineIndexScrubber.ScrubbingPolicy.newBuilder().setLogWarningsLimit(Integer.MAX_VALUE).build()).setTimer(timer).build()) {
        res = indexScrubber.scrubDanglingIndexEntries();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.INDEX_SCRUBBER_DANGLING_ENTRIES));
    assertEquals(0, res);
    // manually delete a few records w/o updating the indexes
    openSimpleMetaData(hook);
    int danglingCount = 0;
    try (FDBRecordContext context = openContext(false)) {
        List<FDBIndexedRecord<Message>> indexRecordEntries = recordStore.scanIndexRecords(tgtIndex.getName(), IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        for (int i = 3; i < numRecords; i *= 2) {
            final FDBIndexedRecord<Message> indexRecord = indexRecordEntries.get(i);
            final FDBStoredRecord<Message> rec = indexRecord.getStoredRecord();
            final Subspace subspace = recordStore.recordsSubspace().subspace(rec.getPrimaryKey());
            recordStore.getContext().ensureActive().clear(subspace.range());
            danglingCount++;
        }
        context.commit();
    }
    // verify the missing entries are found (report only, no repair)
    timer.reset();
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).setScrubbingPolicy(OnlineIndexScrubber.ScrubbingPolicy.newBuilder().setLogWarningsLimit(Integer.MAX_VALUE).setAllowRepair(false)).setTimer(timer).build()) {
        res = indexScrubber.scrubDanglingIndexEntries();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(danglingCount, timer.getCount(FDBStoreTimer.Counts.INDEX_SCRUBBER_DANGLING_ENTRIES));
    assertEquals(danglingCount, res);
    // verify the missing entries are found and fixed
    timer.reset();
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).setScrubbingPolicy(OnlineIndexScrubber.ScrubbingPolicy.newBuilder().setLogWarningsLimit(Integer.MAX_VALUE).build()).setTimer(timer).build()) {
        res = indexScrubber.scrubDanglingIndexEntries();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(danglingCount, timer.getCount(FDBStoreTimer.Counts.INDEX_SCRUBBER_DANGLING_ENTRIES));
    assertEquals(danglingCount, res);
    // if the dangling indexes were removed, this should be reflected later
    numRecords -= danglingCount;
    // now verify it's fixed
    timer.reset();
    try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(tgtIndex).setSubspace(subspace).setScrubbingPolicy(OnlineIndexScrubber.ScrubbingPolicy.newBuilder().setLogWarningsLimit(Integer.MAX_VALUE).build()).setTimer(timer).build()) {
        res = indexScrubber.scrubDanglingIndexEntries();
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.INDEX_SCRUBBER_DANGLING_ENTRIES));
    assertEquals(0, res);
}