Examples with RecordCoreException com.apple.foundationdb.record.RecordCoreException used on opensource projects

Example 31 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerIndexFromIndexTest method testIndexFromIndexOtherSrcIndexWithFallback.

@Test
public void testIndexFromIndexOtherSrcIndexWithFallback() {
    // start indexing by src_index, attempt continue with src_index2
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 83;
    final int chunkSize = 10;
    final int numChunks = 1 + (numRecords / chunkSize);
    Index srcIndex = new Index("src_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    Index srcIndex2 = new Index("src_index2", field("num_value_unique"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    Index tgtIndex = new Index("tgt_index", field("num_value_3_indexed"), IndexTypes.VALUE);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = metaDataBuilder -> {
        metaDataBuilder.addIndex("MySimpleRecord", srcIndex);
        metaDataBuilder.addIndex("MySimpleRecord", srcIndex2);
        metaDataBuilder.addIndex("MySimpleRecord", tgtIndex);
    };
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildSrcIndex(srcIndex);
    openSimpleMetaData(hook);
    buildSrcIndex(srcIndex2);
    // partly build by-index src_index
    openSimpleMetaData(hook);
    buildIndexAndCrashHalfway(tgtIndex, chunkSize, 7, timer, OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().build());
    // try index continuation with src_index2, expect failure
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index2").forbidRecordScan().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).setTimer(timer).build()) {
        RecordCoreException e = assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
        assertTrue(e.getMessage().contains("This index was partly built by another method"));
    }
    // try indexing with src_index2, but allow continuation of previous method (src_index)
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index2").build()).setLimit(chunkSize).setTimer(timer).build()) {
        indexBuilder.buildIndex(true);
    }
    // total of records scan - all by src_index
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(numChunks, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
}

Example 32 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerIndexFromIndexTest method testIndexFromIndexRebuildIfWriteOnlyAndForceBuildAndBuildIfDisabled.

@Test
public void testIndexFromIndexRebuildIfWriteOnlyAndForceBuildAndBuildIfDisabled() {
    // test various policy options to ensure coverage (note that the last section will disable and rebuild the source index)
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 87;
    final int chunkSize = 18;
    final int numChunks = 1 + (numRecords / chunkSize);
    Index srcIndex = new Index("src_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    Index tgtIndex = new Index("tgt_index", field("num_value_3_indexed"), IndexTypes.VALUE);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(srcIndex, tgtIndex);
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildSrcIndex(srcIndex);
    openSimpleMetaData(hook);
    buildIndexAndCrashHalfway(tgtIndex, chunkSize, 3, timer, null);
    openSimpleMetaData(hook);
    timer.reset();
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().setIfWriteOnly(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD).build()).setLimit(chunkSize).setTimer(timer).build()) {
        // now continue building from the last successful range
        indexBuilder.buildIndex(true);
    }
    // counters should demonstrate a continuation to completion
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(numChunks, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
    // now check FORCE_BUILD
    openSimpleMetaData(hook);
    timer.reset();
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().setIfWriteOnly(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD).setIfReadable(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD).build()).setLimit(chunkSize).setTimer(timer).build()) {
        // now force building (but leave it write_only)
        indexBuilder.buildIndex(false);
    }
    // counters should demonstrate a continuation to completion
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(numChunks, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
    // now check BUILD_IF_DISABLED when WRITE_ONLY (from previous test)
    openSimpleMetaData(hook);
    timer.reset();
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfWriteOnly(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).setIfReadable(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR)).setTimer(timer).build()) {
        // now try building if disabled, nothing should be happening
        RecordCoreException e = assertThrows(IndexingBase.ValidationException.class, indexBuilder::buildIndex);
        assertTrue(e.getMessage().contains("Index state is not as expected"));
    }
    assertEquals(0, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    // to test BUILD_IF_DISABLED when disabled - disable the src and build again
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        recordStore.markIndexDisabled(srcIndex).join();
        context.commit();
    }
    openSimpleMetaData(hook);
    try (OnlineIndexer indexer = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(srcIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setIfWriteOnly(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).setIfReadable(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR)).setTimer(timer).build()) {
        indexer.buildIndex(true);
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
}

Example 33 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerIndexFromIndexTest method testIndexFromIndexPersistentContinuePreviousByRecords.

@Test
public void testIndexFromIndexPersistentContinuePreviousByRecords() {
    // start indexing by records, allow continuation of previous by records method - overriding the by-index request
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 90;
    final int chunkSize = 17;
    Index srcIndex = new Index("src_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    Index tgtIndex = new Index("tgt_index", field("num_value_3_indexed"), IndexTypes.VALUE);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(srcIndex, tgtIndex);
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildSrcIndex(srcIndex);
    openSimpleMetaData(hook);
    buildIndexAndCrashHalfway(tgtIndex, chunkSize, 2, timer, null);
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.ERROR).build()).setTimer(timer).build()) {
        // now try building by records, a failure is expected
        RecordCoreException e = assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
        assertTrue(e.getMessage().contains("This index was partly built by another method"));
    }
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setTimer(timer).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().build()).build()) {
        // now continue building, overriding the requested method with the previous one
        indexBuilder.buildIndex(true);
    }
    // counters should demonstrate a continuation to completion
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
}

Example 34 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerIndexFromIndexTest method testIndexFromIndexSrcVersionModifiedWithFallback.

@Test
public void testIndexFromIndexSrcVersionModifiedWithFallback() {
    // start indexing by index, change src index' last modified version, assert failing to continue, continue with REBUILD_IF.. option
    final FDBStoreTimer timer = new FDBStoreTimer();
    final int numRecords = 81;
    final int chunkSize = 10;
    final int numChunks = 1 + (numRecords / chunkSize);
    Index srcIndex = new Index("src_index", field("num_value_2"), EmptyKeyExpression.EMPTY, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
    Index tgtIndex = new Index("tgt_index", field("num_value_3_indexed"), IndexTypes.VALUE);
    FDBRecordStoreTestBase.RecordMetaDataHook hook = myHook(srcIndex, tgtIndex);
    openSimpleMetaData();
    populateData(numRecords);
    openSimpleMetaData(hook);
    buildSrcIndex(srcIndex);
    // partly build by-index
    openSimpleMetaData(hook);
    buildIndexAndCrashHalfway(tgtIndex, chunkSize, 4, timer, OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().build());
    // update src index last modified version (and its subspace key)
    openSimpleMetaData(hook);
    try (FDBRecordContext context = openContext()) {
        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(srcIndex).setSubspace(subspace).build()) {
            recordStore.markIndexWriteOnly(srcIndex).join();
            srcIndex.setLastModifiedVersion(srcIndex.getLastModifiedVersion() + 1);
            indexBuilder.rebuildIndex(recordStore);
            recordStore.markIndexReadable(srcIndex).join();
            context.commit();
        }
    }
    // try index continuation, expect failure
    openSimpleMetaData(hook);
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().build()).setTimer(timer).build()) {
        RecordCoreException e = assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
        assertTrue(e.getMessage().contains("This index was partly built by another method"));
    }
    // try index continuation, but allow rebuild
    openSimpleMetaData(hook);
    timer.reset();
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).addTargetIndex(tgtIndex).setSubspace(subspace).setIndexingPolicy(OnlineIndexer.IndexingPolicy.newBuilder().setSourceIndex("src_index").forbidRecordScan().setIfMismatchPrevious(OnlineIndexer.IndexingPolicy.DesiredAction.REBUILD).build()).setLimit(chunkSize).setTimer(timer).build()) {
        indexBuilder.buildIndex(true);
    }
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_SCANNED));
    assertEquals(numRecords, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RECORDS_INDEXED));
    assertEquals(numChunks, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
}

Example 35 with RecordCoreException

use of com.apple.foundationdb.record.RecordCoreException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerMultiTargetTest method buildIndexAndCrashHalfway.

private void buildIndexAndCrashHalfway(int chunkSize, int count, FDBStoreTimer timer, @Nullable OnlineIndexer.Builder builder) {
    final AtomicLong counter = new AtomicLong(0);
    try (OnlineIndexer indexBuilder = builder.setDatabase(fdb).setMetaData(metaData).setSubspace(subspace).setLimit(chunkSize).setTimer(timer).setConfigLoader(old -> {
        if (counter.incrementAndGet() > count) {
            throw new RecordCoreException("Intentionally crash during test");
        }
        return old;
    }).build()) {
        assertThrows(RecordCoreException.class, indexBuilder::buildIndex);
    // The index should be partially built
    }
    assertEquals(count, timer.getCount(FDBStoreTimer.Counts.ONLINE_INDEX_BUILDER_RANGES_BY_COUNT));
}