Examples with RecordMetaData com.apple.foundationdb.record.RecordMetaData used on opensource projects

Example 51 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class FDBCoveringIndexQueryTest method coveringWithAdditionalNestedFilter.

/**
 * Verify that an extra covering filter can use a nested field.
 */
@DualPlannerTest
void coveringWithAdditionalNestedFilter() {
    try (FDBRecordContext context = openContext()) {
        RecordMetaDataBuilder builder = RecordMetaData.newBuilder().setRecords(TestRecordsWithHeaderProto.getDescriptor());
        builder.getRecordType("MyRecord").setPrimaryKey(field("header").nest(field("rec_no")));
        builder.addIndex("MyRecord", "multi", concat(field("str_value"), field("header").nest(concatenateFields("path", "num"))));
        RecordMetaData metaData = builder.getRecordMetaData();
        createOrOpenRecordStore(context, metaData);
        RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.and(Query.field("str_value").equalsValue("abc"), Query.field("header").matches(Query.field("num").equalsValue(1)))).build();
        // Fetch(Covering(Index(multi [[abc],[abc]]) -> [str_value: KEY[0], header: [num: KEY[2], path: KEY[1], rec_no: KEY[3]]]) | header/{num EQUALS 1})
        RecordQueryPlan plan = planner.plan(query);
        if (planner instanceof RecordQueryPlanner) {
            final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(filterPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi")).and(scanComparisons(range("[[abc],[abc]]")))))).where(queryComponents(exactly(equalsObject(Query.field("header").matches(Query.field("num").equalsValue(1)))))));
            assertMatchesExactly(plan, planMatcher);
            assertEquals(-1536005152, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
            assertEquals(1350035332, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
            assertEquals(-1843652335, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        } else {
            final BindingMatcher<? extends RecordQueryPlan> planMatcher = fetchFromPartialRecordPlan(predicatesFilterPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("multi")).and(scanComparisons(range("[[abc],[abc]]")))))).where(predicates(only(valuePredicate(fieldValue("header.num"), new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, 1))))));
            assertMatchesExactly(plan, planMatcher);
            assertEquals(1623341655, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
            assertEquals(2019556616, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
            assertEquals(-1174131051, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
        }
    }
}

Example 52 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method conflictWithHeaderChange.

/**
 * Validate that if the store header changes then an open record store in another transaction is failed with
 * a conflict.
 */
@Test
public void conflictWithHeaderChange() {
    RecordMetaData metaData1 = RecordMetaData.build(TestRecords1Proto.getDescriptor());
    RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
    metaDataBuilder.addIndex("MySimpleRecord", "num_value_2");
    RecordMetaData metaData2 = metaDataBuilder.getRecordMetaData();
    assertThat(metaData1.getVersion(), lessThan(metaData2.getVersion()));
    try (FDBRecordContext context = openContext()) {
        FDBRecordStore recordStore = storeBuilder(context, metaData1).create();
        assertEquals(metaData1.getVersion(), recordStore.getRecordStoreState().getStoreHeader().getMetaDataversion());
        commit(context);
    }
    try (FDBRecordContext context1 = openContext();
        FDBRecordContext context2 = openContext()) {
        FDBRecordStore recordStore1 = storeBuilder(context1, metaData1).open();
        assertEquals(metaData1.getVersion(), recordStore1.getRecordStoreState().getStoreHeader().getMetaDataversion());
        FDBRecordStore recordStore2 = storeBuilder(context2, metaData2).open();
        assertEquals(metaData2.getVersion(), recordStore2.getRecordStoreState().getStoreHeader().getMetaDataversion());
        commit(context2);
        // Add a write to the first record store to make sure that the conflict are actually checked.
        recordStore1.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(1415).build());
        assertThrows(FDBExceptions.FDBStoreTransactionConflictException.class, context1::commit);
    }
    try (FDBRecordContext context = openContext()) {
        assertThrows(RecordStoreStaleMetaDataVersionException.class, () -> storeBuilder(context, metaData1).open());
        FDBRecordStore recordStore = storeBuilder(context, metaData2).open();
        assertEquals(metaData2.getVersion(), recordStore.getRecordStoreState().getStoreHeader().getMetaDataversion());
        commit(context);
    }
}

Example 53 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method storeExistenceChecksWithNoRecords.

@Test
public void storeExistenceChecksWithNoRecords() throws Exception {
    RecordMetaData metaData = RecordMetaData.build(TestRecords1Proto.getDescriptor());
    FDBRecordStore.Builder storeBuilder;
    try (FDBRecordContext context = openContext()) {
        storeBuilder = storeBuilder(context, metaData);
        FDBRecordStore store = storeBuilder.create();
        // delete the header
        store.ensureContextActive().clear(getStoreInfoKey(store));
        commit(context);
    }
    // Should be able to recover from a completely empty record store
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        FDBRecordStore store = storeBuilder.createOrOpen();
        // put a range subspace in for an index so that a future store opening can see it
        store.ensureContextActive().clear(getStoreInfoKey(store));
        Index foundIndex = metaData.getAllIndexes().stream().findAny().orElseGet(() -> fail("no indexes defined in meta-data"));
        new RangeSet(store.indexRangeSubspace(foundIndex)).insertRange(context.ensureActive(), null, null).get();
        // re-delete the header
        store.ensureContextActive().clear(getStoreInfoKey(store));
        commit(context);
    }
    // should be recoverable using ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        // do not perform checkVersion yet
        FDBRecordStore store = storeBuilder.build();
        assertNull(context.ensureActive().get(getStoreInfoKey(store)).get());
        assertTrue(store.checkVersion(null, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES).get());
        commit(context);
    }
    // Delete everything except a value in the index build space
    try (FDBRecordContext context = openContext()) {
        FDBRecordStore store = storeBuilder.setContext(context).open();
        final Subspace subspace = OnlineIndexer.indexBuildScannedRecordsSubspace(store, metaData.getIndex("MySimpleRecord$str_value_indexed"));
        // set a key in the INDEX_BUILD_SPACE
        context.ensureActive().set(subspace.getKey(), FDBRecordStore.encodeRecordCount(1215));
        context.ensureActive().clear(store.getSubspace().getKey(), subspace.getKey());
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context).createOrOpen(FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES);
        commit(context);
    }
    // Insert a record, then delete the store header
    try (FDBRecordContext context = openContext()) {
        // open as the previous open with the relaxed existence check should have fixed the store header
        FDBRecordStore store = storeBuilder.setContext(context).open();
        store.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).build());
        store.ensureContextActive().clear(getStoreInfoKey(store));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, () -> storeBuilder.createOrOpen(FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES));
        commit(context);
    }
    // Delete the record store, then insert a key at an unknown keyspace
    try (FDBRecordContext context = openContext()) {
        FDBRecordStore.deleteStore(context, path);
        Subspace subspace = path.toSubspace(context);
        context.ensureActive().set(subspace.pack("unknown_keyspace"), Tuple.from("doesn't matter").pack());
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        storeBuilder.setContext(context);
        assertThrows(RecordStoreNoInfoAndNotEmptyException.class, storeBuilder::createOrOpen);
        RecordCoreException err = assertThrows(RecordCoreException.class, () -> storeBuilder.createOrOpen(FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NO_INFO_AND_HAS_RECORDS_OR_INDEXES));
        assertEquals("Unrecognized keyspace: unknown_keyspace", err.getMessage());
        commit(context);
    }
}

Example 54 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method sizeBasedUserVersionChecker.

@Test
public void sizeBasedUserVersionChecker() {
    final Index universalCountIndex = new Index("countIndex", new GroupingKeyExpression(EmptyKeyExpression.EMPTY, 0), IndexTypes.COUNT);
    final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
    metaDataBuilder.addUniversalIndex(universalCountIndex);
    final RecordMetaData metaData1 = metaDataBuilder.getRecordMetaData();
    // Open the store and make sure estimate size path is used
    try (FDBRecordContext context = openContext()) {
        final SizeBasedUserVersionChecker userVersionChecker = new SizeBasedUserVersionChecker(IndexState.READABLE);
        final FDBRecordStore recordStore = storeBuilder(context, metaData1).setUserVersionChecker(userVersionChecker).create();
        assertThat("should have checked the size at least once", userVersionChecker.getCheckSizeCount(), greaterThan(0L));
        assertEquals(1, timer.getCount(FDBStoreTimer.Events.ESTIMATE_SIZE));
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable(), "all indexes should be readable on new store opening");
        commit(context);
    }
    // Re-open the store and make sure the estimate is never used
    try (FDBRecordContext context = openContext()) {
        final SizeBasedUserVersionChecker userVersionChecker = new SizeBasedUserVersionChecker(IndexState.DISABLED);
        final FDBRecordStore recordStore = storeBuilder(context, metaData1).setUserVersionChecker(userVersionChecker).open();
        assertEquals(0, userVersionChecker.getCheckSizeCount(), "should not have checked the size on already created store");
        assertEquals(0, timer.getCount(FDBStoreTimer.Events.ESTIMATE_SIZE), "should not have estimated the size on already created store");
        assertTrue(recordStore.getRecordStoreState().allIndexesReadable(), "all indexes should be readable on already created store opening");
        commit(context);
    }
    // Add two indexes
    final String indexName1 = "index1";
    final String indexName2 = "index2";
    metaDataBuilder.addIndex("MySimpleRecord", indexName1, "num_value_2");
    metaDataBuilder.addIndex("MySimpleRecord", indexName2, "num_value_2");
    final RecordMetaData metaData2 = metaDataBuilder.getRecordMetaData();
    try (FDBRecordContext context = openContext()) {
        final SizeBasedUserVersionChecker userVersionChecker = new SizeBasedUserVersionChecker(IndexState.DISABLED);
        final FDBRecordStore recordStore = storeBuilder(context, metaData2).setUserVersionChecker(userVersionChecker).open();
        assertEquals(2, userVersionChecker.getCheckSizeCount(), "should have checked the size once for each index");
        assertEquals(1, timer.getCount(FDBStoreTimer.Events.ESTIMATE_SIZE), "should have only checked the database for the size once");
        assertEquals(ImmutableSet.of(indexName1, indexName2), recordStore.getRecordStoreState().getDisabledIndexNames());
        commit(context);
    }
}

Example 55 with RecordMetaData

use of com.apple.foundationdb.record.RecordMetaData in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreOpeningTest method testUpdateRecords.

@Test
public void testUpdateRecords() {
    KeySpacePath metaDataPath;
    Subspace metaDataSubspace;
    try (FDBRecordContext context = fdb.openContext()) {
        metaDataPath = TestKeySpace.getKeyspacePath("record-test", "unit", "metadataStore");
        metaDataSubspace = metaDataPath.toSubspace(context);
        context.ensureActive().clear(Range.startsWith(metaDataSubspace.pack()));
        context.commit();
    }
    try (FDBRecordContext context = fdb.openContext()) {
        RecordMetaData origMetaData = RecordMetaData.build(TestRecords1Proto.getDescriptor());
        final int version = origMetaData.getVersion();
        FDBMetaDataStore metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1Proto.getDescriptor());
        FDBRecordStore recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).createOrOpen();
        assertEquals(version, recordStore.getRecordMetaData().getVersion());
        TestRecords1Proto.MySimpleRecord record = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).setStrValueIndexed("value").setNumValue3Indexed(1729).build();
        recordStore.saveRecord(record);
        // Update the records without a local descriptor. Storing an evolved record must fail.
        final TestRecords1EvolvedProto.MySimpleRecord evolvedRecord = TestRecords1EvolvedProto.MySimpleRecord.newBuilder().setRecNo(1067L).setNumValue2(43).setStrValueIndexed("evolved value").setNumValue3Indexed(1730).build();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, null);
        // Bumps the version
        metaDataStore.updateRecords(TestRecords1EvolvedProto.getDescriptor());
        final FDBRecordStore recordStoreWithNoLocalFileDescriptor = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(version + 1, recordStoreWithNoLocalFileDescriptor.getRecordMetaData().getVersion());
        MetaDataException e = assertThrows(MetaDataException.class, () -> recordStoreWithNoLocalFileDescriptor.saveRecord(evolvedRecord));
        assertEquals(e.getMessage(), "descriptor did not match record type");
        // Update the records with a local descriptor. Storing an evolved record must succeed this time.
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1EvolvedProto.getDescriptor());
        // Bumps the version
        metaDataStore.updateRecords(TestRecords1EvolvedProto.getDescriptor());
        recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(version + 2, recordStore.getRecordMetaData().getVersion());
        recordStore.saveRecord(evolvedRecord);
        // Evolve the meta-data one more time and use it for local file descriptor. SaveRecord will succeed.
        final TestRecords1EvolvedAgainProto.MySimpleRecord evolvedAgainRecord = TestRecords1EvolvedAgainProto.MySimpleRecord.newBuilder().setRecNo(1066L).setNumValue2(42).setStrValueIndexed("value").setNumValue3Indexed(1729).build();
        metaDataStore = createMetaDataStore(context, metaDataPath, metaDataSubspace, TestRecords1EvolvedAgainProto.getDescriptor());
        // Bumps the version
        metaDataStore.updateRecords(TestRecords1EvolvedProto.getDescriptor());
        recordStore = storeBuilder(context, origMetaData).setMetaDataStore(metaDataStore).open();
        assertEquals(version + 3, recordStore.getRecordMetaData().getVersion());
        recordStore.saveRecord(evolvedAgainRecord);
    }
}